include parsing files

ef-migration/infrastructure/docker-compose-infra.yml

@@ -15,6 +15,7 @@ services:
       - DROP_DATABASE=${DROP_DATABASE:-false} 
     volumes:
       - sqlserver_data:/var/opt/mssql
+      - ./logs:/usr/src/app/logs
     deploy:
       resources:
         limits:

ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Check if input file is provided
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <sql_file>"
+    exit 1
+fi
+
+SQL_FILE="$1"
+BASE_DIR="sql_objects"
+
+# Create directory structure
+mkdir -p "$BASE_DIR"/{constraints,foreign_keys}
+
+# Function to clean filename
+clean_filename() {
+    # Remove brackets, convert dots and spaces to underscores, remove parentheses
+    echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
+}
+
+# Function to extract table name from constraint
+extract_table_name() {
+    echo "$1" | sed -nE 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^]]+)\]?.*/\1/p'
+}
+
+# Function to extract constraint name
+extract_constraint_name() {
+    echo "$1" | sed -nE 's/.*CONSTRAINT[[:space:]]+\[?([^]]+)\]?.*/\1/p'
+}
+
+# Function to process each ALTER TABLE statement
+process_statement() {
+    local line="$1"
+    local table_name=$(extract_table_name "$line")
+    local constraint_name=$(extract_constraint_name "$line")
+    
+    # Skip if we couldn't extract table or constraint name
+    if [ -z "$table_name" ] || [ -z "$constraint_name" ]; then
+        return
+    fi
+    
+    # Determine if it's a foreign key
+    if echo "$line" | grep -i "FOREIGN[[:space:]]\+KEY" >/dev/null; then
+        output_dir="$BASE_DIR/foreign_keys"
+    else
+        output_dir="$BASE_DIR/constraints"
+    fi
+    
+    # Clean filename and save
+    clean_name=$(clean_filename "${table_name}_${constraint_name}")
+    output_file="${output_dir}/${clean_name}.sql"
+    echo "$line" > "$output_file"
+    echo "Created: $output_file"
+}
+
+# Main processing function
+extract_constraints() {
+    local current_statement=""
+    
+    # Read the file line by line
+    while IFS= read -r line || [ -n "$line" ]; do
+        # Skip empty lines
+        if [ -z "${line// }" ]; then
+            continue
+        fi
+        
+        # If line starts with ALTER TABLE, it's a new statement
+        if echo "$line" | grep -i "^[[:space:]]*ALTER[[:space:]]\+TABLE" >/dev/null; then
+            current_statement="$line"
+            
+            # If GO is on the same line
+            if echo "$line" | grep -i "GO[[:space:]]*$" >/dev/null; then
+                # Remove GO and process
+                current_statement=$(echo "$current_statement" | sed 's/[[:space:]]*GO[[:space:]]*$//')
+                process_statement "$current_statement"
+                current_statement=""
+            fi
+            continue
+        fi
+        
+        # If we find GO, process the current statement
+        if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
+            if [ -n "$current_statement" ]; then
+                process_statement "$current_statement"
+                current_statement=""
+            fi
+            continue
+        fi
+        
+        # Add line to current statement if we're collecting one
+        if [ -n "$current_statement" ]; then
+            current_statement+=" $line"
+        fi
+    done
+}
+
+# Remove comment lines starting with -- and process
+echo "Extracting constraints and foreign keys..."
+sed 's/--.*$//' "$SQL_FILE" | extract_constraints
+
+echo "Constraints and foreign keys have been saved in $BASE_DIR/constraints and $BASE_DIR/foreign_keys"
+
+# Count and list files
+echo -e "\nFile counts:"
+for dir in constraints foreign_keys; do
+    count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l || echo 0)
+    echo "$dir: $count files"
+    
+    echo -e "\nExtracted ${dir}:"
+    if [ "$count" -gt 0 ]; then
+        for file in "$BASE_DIR/$dir"/*.sql; do
+            if [ -f "$file" ]; then
+                echo "- $(basename "$file")"
+            fi
+        done
+    fi
+done

ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Check if input file is provided
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <sql_file>"
+    exit 1
+fi
+
+SQL_FILE="$1"
+BASE_DIR="sql_objects"
+
+# Create directory structure
+mkdir -p "$BASE_DIR/tables"
+
+# Function to clean filename
+clean_filename() {
+    # Remove brackets, convert dots and spaces to underscores, remove parentheses
+    echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
+}
+
+# Function to check if table is temporary
+is_temporary_table() {
+    local table_name="$1"
+    if echo "$table_name" | grep -iE "temp|tmp|temporary|import_|viewimport_" >/dev/null; then
+        return 0 # true in bash
+    else
+        return 1 # false in bash
+    fi
+}
+
+# Function to extract CREATE TABLE statements only
+extract_tables() {
+    local current_object=""
+    local capture_table=false
+    local table_name=""
+    
+    # Read the file line by line
+    while IFS= read -r line || [ -n "$line" ]; do
+        # Check if we've found a CREATE TABLE statement
+        if echo "$line" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then
+            # Extract table name and check if it's temporary
+            table_name=$(echo "$line" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+            
+            if is_temporary_table "$table_name"; then
+                echo "Skipping temporary table: $table_name"
+                continue
+            fi
+            
+            capture_table=true
+            current_object="$line"$'\n'
+            continue
+        fi
+
+        # If we're capturing a table and find a following CREATE or ALTER statement, stop capturing
+        if $capture_table; then
+            if echo "$line" | grep -i "^[[:space:]]*CREATE\|^[[:space:]]*ALTER" >/dev/null; then
+                # Save the current table before starting a new object
+                if [ -n "$table_name" ]; then
+                    clean_name=$(clean_filename "$table_name")
+                    output_file="$BASE_DIR/tables/${clean_name}.sql"
+                    echo "$current_object" > "$output_file"
+                    echo "Created: $output_file"
+                fi
+                capture_table=false
+                current_object=""
+                table_name=""
+                continue
+            fi
+
+            # Add line to current object if we're still capturing
+            current_object+="$line"$'\n'
+
+            # If we find a GO statement, save the table
+            if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
+                if [ -n "$table_name" ]; then
+                    clean_name=$(clean_filename "$table_name")
+                    output_file="$BASE_DIR/tables/${clean_name}.sql"
+                    echo "$current_object" > "$output_file"
+                    echo "Created: $output_file"
+                fi
+                capture_table=false
+                current_object=""
+                table_name=""
+            fi
+        fi
+    done < "$SQL_FILE"
+}
+
+# Remove comment lines starting with --
+echo "Extracting CREATE TABLE statements..."
+sed 's/--.*$//' "$SQL_FILE" | extract_tables
+
+echo "CREATE TABLE statements have been saved in $BASE_DIR/tables/"
+
+# Count files
+count=$(ls -1 "$BASE_DIR/tables"/*.sql 2>/dev/null | wc -l)
+echo -e "\nTotal tables extracted: $count"
+
+# List extracted tables
+echo -e "\nExtracted tables:"
+for table in "$BASE_DIR/tables"/*.sql; do
+    if [ -f "$table" ]; then
+        echo "- $(basename "$table")"
+    fi
+done

ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+# Check if input file is provided
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <sql_file>"
+    exit 1
+fi
+
+SQL_FILE="$1"
+BASE_DIR="sql_objects"
+
+# Create directory structure
+mkdir -p "$BASE_DIR/views"
+
+# Function to clean filename
+clean_filename() {
+    # Remove brackets, convert dots and spaces to underscores, remove parentheses
+    echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
+}
+
+# Function to extract view name
+extract_view_name() {
+    # More flexible pattern for extracting view name
+    echo "$1" | sed -n 's/.*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]*[Vv][Ii][Ee][Ww][[:space:]]*\[\([^]]*\)\]\.\[\([^]]*\)\].*/\1.\2/p'
+}
+
+# Function to clean the input line
+clean_input_line() {
+    # Remove any leading/trailing whitespace and normalize spaces
+    echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | tr -s ' '
+}
+
+# Function to process each CREATE VIEW statement
+process_view() {
+    local current_object="$1"
+    local view_name="$2"
+    
+    # Skip if we couldn't extract view name
+    if [ -z "$view_name" ]; then
+        echo "Warning: Could not extract view name from statement:"
+        echo "$current_object" | head -n 1
+        # Try to extract with a more lenient pattern
+        view_name=$(echo "$current_object" | head -n 1 | grep -io '\[int\]\.\[[^]]*\]' | head -n 1 | sed 's/\[\|\]//g')
+        if [ -z "$view_name" ]; then
+            return
+        fi
+    fi
+    
+    # Clean filename and save
+    clean_name=$(clean_filename "$view_name")
+    output_file="$BASE_DIR/views/${clean_name}.sql"
+    echo "$current_object" > "$output_file"
+    echo "Created: $output_file (from view: $view_name)"
+}
+
+# Main processing function
+extract_views() {
+    local current_object=""
+    local capture_view=false
+    local view_name=""
+    
+    # Read the file line by line
+    while IFS= read -r line || [ -n "$line" ]; do
+        # Clean the line
+        line=$(clean_input_line "$line")
+        
+        # Skip empty lines at the start of an object
+        if [ -z "$line" ] && [ -z "$current_object" ]; then
+            continue
+        fi
+
+        # Check if we've found a CREATE VIEW statement (case insensitive and flexible)
+        if echo "$line" | grep -iE "[[:space:]]*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]+[Vv][Ii][Ee][Ww]" >/dev/null; then
+            capture_view=true
+            current_object="$line"
+            view_name=$(extract_view_name "$line")
+            if [ -n "$view_name" ]; then
+                echo "Found view: $view_name"
+            fi
+            continue
+        fi
+
+        # If we're capturing a view
+        if $capture_view; then
+            # Check if we've reached the end (GO statement)
+            if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
+                # Add GO to the view definition
+                current_object+=$'\n'"GO"
+                process_view "$current_object" "$view_name"
+                capture_view=false
+                current_object=""
+                view_name=""
+                continue
+            fi
+
+            # Add line to current object
+            current_object+=$'\n'"$line"
+        fi
+    done
+
+    # Handle case where last view might not have a GO statement
+    if $capture_view && [ -n "$current_object" ]; then
+        process_view "$current_object" "$view_name"
+    fi
+}
+
+# Pre-process the file to normalize line endings and remove comments
+echo "Extracting views..."
+sed 's/--.*$//; /^[[:space:]]*$/d' "$SQL_FILE" | extract_views
+
+echo "Views have been saved in $BASE_DIR/views"
+
+# Count and list files
+count=$(ls -1 "$BASE_DIR/views"/*.sql 2>/dev/null | wc -l || echo 0)
+echo -e "\nTotal views extracted: $count"
+
+if [ "$count" -gt 0 ]; then
+    echo -e "\nExtracted views:"
+    for file in "$BASE_DIR/views"/*.sql; do
+        if [ -f "$file" ]; then
+            echo "- $(basename "$file")"
+        fi
+    done
+fi

ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Check if input file is provided
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <sql_file>"
+    exit 1
+fi
+
+SQL_FILE="$1"
+BASE_DIR="sql_objects"
+
+# Create directory structure
+mkdir -p "$BASE_DIR"/{tables,views,procedures,constraints,indexes,alter_tables}
+
+# Function to clean filename
+clean_filename() {
+    echo "$1" | sed 's/\[\|\]//g' | tr '.' '_'
+}
+
+# Function to extract SQL objects and save to files
+extract_objects() {
+    local current_object=""
+    local object_type=""
+    local object_name=""
+    local table_name=""
+    
+    # Read the file line by line
+    while IFS= read -r line || [ -n "$line" ]; do
+        # Skip empty lines at the start
+        if [ -z "$current_object" ] && [ -z "${line// }" ]; then
+            continue
+        fi
+
+        # Add line to current object
+        current_object+="$line"$'\n'
+
+        # Check if we've reached the end of an object (GO statement)
+        if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
+            # Determine object type and name
+            if echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then
+                object_type="tables"
+                object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+            elif echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" >/dev/null; then
+                object_type="views"
+                object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" | sed -E 's/.*CREATE[[:space:]]+VIEW[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+            elif echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC" >/dev/null; then
+                object_type="procedures"
+                object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC[[:space:]EDURE]*" | sed -E 's/.*CREATE[[:space:]]+PROC[[:space:]EDURE]*[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+            elif echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" >/dev/null; then
+                # Extract table name and constraint name if it exists
+                table_name=$(echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" | sed -E 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+                
+                if echo "$current_object" | grep -i "ADD[[:space:]]\+CONSTRAINT" >/dev/null; then
+                    object_type="constraints"
+                    object_name=$(echo "$current_object" | grep -i "CONSTRAINT" | sed -E 's/.*CONSTRAINT[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+                else
+                    object_type="alter_tables"
+                    object_name="${table_name}_alter_$(date +%s%N | cut -b1-5)"
+                fi
+            elif echo "$current_object" | grep -i "CREATE.*INDEX" >/dev/null; then
+                object_type="indexes"
+                object_name=$(echo "$current_object" | grep -i "INDEX" | sed -E 's/.*INDEX[[:space:]]+\[?([^\]]+)\]?.*/\1/')
+            fi
+
+            # If we identified an object type and name, save it
+            if [ -n "$object_type" ] && [ -n "$object_name" ]; then
+                # Clean the filename
+                clean_name=$(clean_filename "$object_name")
+                output_file="$BASE_DIR/$object_type/$clean_name.sql"
+
+                # For alter_tables, make sure we don't overwrite existing files
+                if [ "$object_type" = "alter_tables" ]; then
+                    counter=1
+                    base_file="$BASE_DIR/$object_type/$clean_name"
+                    while [ -f "${base_file}.sql" ]; do
+                        clean_name="${clean_name}_${counter}"
+                        counter=$((counter + 1))
+                    done
+                    output_file="$base_file.sql"
+                fi
+
+                # Save the object to file
+                echo "$current_object" > "$output_file"
+                echo "Created: $output_file"
+            fi
+
+            # Reset for next object
+            current_object=""
+            object_type=""
+            object_name=""
+            table_name=""
+        fi
+    done < "$SQL_FILE"
+}
+
+# Remove comment lines starting with --
+echo "Removing comments and processing SQL file..."
+sed 's/--.*$//' "$SQL_FILE" | extract_objects
+
+echo "SQL objects have been organized in the $BASE_DIR directory."
+echo "
+Directory structure:
+$BASE_DIR/
+├── tables/
+├── views/
+├── procedures/
+├── constraints/
+├── indexes/
+└── alter_tables/"
+
+# Count files in each directory
+echo -e "\nFile counts:"
+for dir in tables views procedures constraints indexes alter_tables; do
+    count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l)
+    echo "$dir: $count files"
+done