diff --git a/ef-migration/infrastructure/docker-compose-infra.yml b/ef-migration/infrastructure/docker-compose-infra.yml index c51a07a..3f59a15 100644 --- a/ef-migration/infrastructure/docker-compose-infra.yml +++ b/ef-migration/infrastructure/docker-compose-infra.yml @@ -15,6 +15,7 @@ services: - DROP_DATABASE=${DROP_DATABASE:-false} volumes: - sqlserver_data:/var/opt/mssql + - ./logs:/usr/src/app/logs deploy: resources: limits: diff --git a/ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh b/ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh new file mode 100755 index 0000000..a4ce1d8 --- /dev/null +++ b/ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Check if input file is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +SQL_FILE="$1" +BASE_DIR="sql_objects" + +# Create directory structure +mkdir -p "$BASE_DIR"/{constraints,foreign_keys} + +# Function to clean filename +clean_filename() { + # Remove brackets, convert dots and spaces to underscores, remove parentheses + echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//' +} + +# Function to extract table name from constraint +extract_table_name() { + echo "$1" | sed -nE 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^]]+)\]?.*/\1/p' +} + +# Function to extract constraint name +extract_constraint_name() { + echo "$1" | sed -nE 's/.*CONSTRAINT[[:space:]]+\[?([^]]+)\]?.*/\1/p' +} + +# Function to process each ALTER TABLE statement +process_statement() { + local line="$1" + local table_name=$(extract_table_name "$line") + local constraint_name=$(extract_constraint_name "$line") + + # Skip if we couldn't extract table or constraint name + if [ -z "$table_name" ] || [ -z "$constraint_name" ]; then + return + fi + + # Determine if it's a foreign key + if echo "$line" | grep -i "FOREIGN[[:space:]]\+KEY" >/dev/null; then + output_dir="$BASE_DIR/foreign_keys" + else + output_dir="$BASE_DIR/constraints" + fi + + # Clean filename and save + clean_name=$(clean_filename "${table_name}_${constraint_name}") + output_file="${output_dir}/${clean_name}.sql" + echo "$line" > "$output_file" + echo "Created: $output_file" +} + +# Main processing function +extract_constraints() { + local current_statement="" + + # Read the file line by line + while IFS= read -r line || [ -n "$line" ]; do + # Skip empty lines + if [ -z "${line// }" ]; then + continue + fi + + # If line starts with ALTER TABLE, it's a new statement + if echo "$line" | grep -i "^[[:space:]]*ALTER[[:space:]]\+TABLE" >/dev/null; then + current_statement="$line" + + # If GO is on the same line + if echo "$line" | grep -i "GO[[:space:]]*$" >/dev/null; then + # Remove GO and process + current_statement=$(echo "$current_statement" | sed 's/[[:space:]]*GO[[:space:]]*$//') + process_statement "$current_statement" + current_statement="" + fi + continue + fi + + # If we find GO, process the current statement + if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then + if [ -n "$current_statement" ]; then + process_statement "$current_statement" + current_statement="" + fi + continue + fi + + # Add line to current statement if we're collecting one + if [ -n "$current_statement" ]; then + current_statement+=" $line" + fi + done +} + +# Remove comment lines starting with -- and process +echo "Extracting constraints and foreign keys..." +sed 's/--.*$//' "$SQL_FILE" | extract_constraints + +echo "Constraints and foreign keys have been saved in $BASE_DIR/constraints and $BASE_DIR/foreign_keys" + +# Count and list files +echo -e "\nFile counts:" +for dir in constraints foreign_keys; do + count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l || echo 0) + echo "$dir: $count files" + + echo -e "\nExtracted ${dir}:" + if [ "$count" -gt 0 ]; then + for file in "$BASE_DIR/$dir"/*.sql; do + if [ -f "$file" ]; then + echo "- $(basename "$file")" + fi + done + fi +done \ No newline at end of file diff --git a/ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh b/ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh new file mode 100755 index 0000000..d9959d2 --- /dev/null +++ b/ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Check if input file is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +SQL_FILE="$1" +BASE_DIR="sql_objects" + +# Create directory structure +mkdir -p "$BASE_DIR/tables" + +# Function to clean filename +clean_filename() { + # Remove brackets, convert dots and spaces to underscores, remove parentheses + echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//' +} + +# Function to check if table is temporary +is_temporary_table() { + local table_name="$1" + if echo "$table_name" | grep -iE "temp|tmp|temporary|import_|viewimport_" >/dev/null; then + return 0 # true in bash + else + return 1 # false in bash + fi +} + +# Function to extract CREATE TABLE statements only +extract_tables() { + local current_object="" + local capture_table=false + local table_name="" + + # Read the file line by line + while IFS= read -r line || [ -n "$line" ]; do + # Check if we've found a CREATE TABLE statement + if echo "$line" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then + # Extract table name and check if it's temporary + table_name=$(echo "$line" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/') + + if is_temporary_table "$table_name"; then + echo "Skipping temporary table: $table_name" + continue + fi + + capture_table=true + current_object="$line"$'\n' + continue + fi + + # If we're capturing a table and find a following CREATE or ALTER statement, stop capturing + if $capture_table; then + if echo "$line" | grep -i "^[[:space:]]*CREATE\|^[[:space:]]*ALTER" >/dev/null; then + # Save the current table before starting a new object + if [ -n "$table_name" ]; then + clean_name=$(clean_filename "$table_name") + output_file="$BASE_DIR/tables/${clean_name}.sql" + echo "$current_object" > "$output_file" + echo "Created: $output_file" + fi + capture_table=false + current_object="" + table_name="" + continue + fi + + # Add line to current object if we're still capturing + current_object+="$line"$'\n' + + # If we find a GO statement, save the table + if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then + if [ -n "$table_name" ]; then + clean_name=$(clean_filename "$table_name") + output_file="$BASE_DIR/tables/${clean_name}.sql" + echo "$current_object" > "$output_file" + echo "Created: $output_file" + fi + capture_table=false + current_object="" + table_name="" + fi + fi + done < "$SQL_FILE" +} + +# Remove comment lines starting with -- +echo "Extracting CREATE TABLE statements..." +sed 's/--.*$//' "$SQL_FILE" | extract_tables + +echo "CREATE TABLE statements have been saved in $BASE_DIR/tables/" + +# Count files +count=$(ls -1 "$BASE_DIR/tables"/*.sql 2>/dev/null | wc -l) +echo -e "\nTotal tables extracted: $count" + +# List extracted tables +echo -e "\nExtracted tables:" +for table in "$BASE_DIR/tables"/*.sql; do + if [ -f "$table" ]; then + echo "- $(basename "$table")" + fi +done \ No newline at end of file diff --git a/ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh b/ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh new file mode 100755 index 0000000..a708894 --- /dev/null +++ b/ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# Check if input file is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +SQL_FILE="$1" +BASE_DIR="sql_objects" + +# Create directory structure +mkdir -p "$BASE_DIR/views" + +# Function to clean filename +clean_filename() { + # Remove brackets, convert dots and spaces to underscores, remove parentheses + echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//' +} + +# Function to extract view name +extract_view_name() { + # More flexible pattern for extracting view name + echo "$1" | sed -n 's/.*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]*[Vv][Ii][Ee][Ww][[:space:]]*\[\([^]]*\)\]\.\[\([^]]*\)\].*/\1.\2/p' +} + +# Function to clean the input line +clean_input_line() { + # Remove any leading/trailing whitespace and normalize spaces + echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | tr -s ' ' +} + +# Function to process each CREATE VIEW statement +process_view() { + local current_object="$1" + local view_name="$2" + + # Skip if we couldn't extract view name + if [ -z "$view_name" ]; then + echo "Warning: Could not extract view name from statement:" + echo "$current_object" | head -n 1 + # Try to extract with a more lenient pattern + view_name=$(echo "$current_object" | head -n 1 | grep -io '\[int\]\.\[[^]]*\]' | head -n 1 | sed 's/\[\|\]//g') + if [ -z "$view_name" ]; then + return + fi + fi + + # Clean filename and save + clean_name=$(clean_filename "$view_name") + output_file="$BASE_DIR/views/${clean_name}.sql" + echo "$current_object" > "$output_file" + echo "Created: $output_file (from view: $view_name)" +} + +# Main processing function +extract_views() { + local current_object="" + local capture_view=false + local view_name="" + + # Read the file line by line + while IFS= read -r line || [ -n "$line" ]; do + # Clean the line + line=$(clean_input_line "$line") + + # Skip empty lines at the start of an object + if [ -z "$line" ] && [ -z "$current_object" ]; then + continue + fi + + # Check if we've found a CREATE VIEW statement (case insensitive and flexible) + if echo "$line" | grep -iE "[[:space:]]*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]+[Vv][Ii][Ee][Ww]" >/dev/null; then + capture_view=true + current_object="$line" + view_name=$(extract_view_name "$line") + if [ -n "$view_name" ]; then + echo "Found view: $view_name" + fi + continue + fi + + # If we're capturing a view + if $capture_view; then + # Check if we've reached the end (GO statement) + if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then + # Add GO to the view definition + current_object+=$'\n'"GO" + process_view "$current_object" "$view_name" + capture_view=false + current_object="" + view_name="" + continue + fi + + # Add line to current object + current_object+=$'\n'"$line" + fi + done + + # Handle case where last view might not have a GO statement + if $capture_view && [ -n "$current_object" ]; then + process_view "$current_object" "$view_name" + fi +} + +# Pre-process the file to normalize line endings and remove comments +echo "Extracting views..." +sed 's/--.*$//; /^[[:space:]]*$/d' "$SQL_FILE" | extract_views + +echo "Views have been saved in $BASE_DIR/views" + +# Count and list files +count=$(ls -1 "$BASE_DIR/views"/*.sql 2>/dev/null | wc -l || echo 0) +echo -e "\nTotal views extracted: $count" + +if [ "$count" -gt 0 ]; then + echo -e "\nExtracted views:" + for file in "$BASE_DIR/views"/*.sql; do + if [ -f "$file" ]; then + echo "- $(basename "$file")" + fi + done +fi \ No newline at end of file diff --git a/ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh b/ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh new file mode 100755 index 0000000..afa86a4 --- /dev/null +++ b/ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# Check if input file is provided +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +SQL_FILE="$1" +BASE_DIR="sql_objects" + +# Create directory structure +mkdir -p "$BASE_DIR"/{tables,views,procedures,constraints,indexes,alter_tables} + +# Function to clean filename +clean_filename() { + echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' +} + +# Function to extract SQL objects and save to files +extract_objects() { + local current_object="" + local object_type="" + local object_name="" + local table_name="" + + # Read the file line by line + while IFS= read -r line || [ -n "$line" ]; do + # Skip empty lines at the start + if [ -z "$current_object" ] && [ -z "${line// }" ]; then + continue + fi + + # Add line to current object + current_object+="$line"$'\n' + + # Check if we've reached the end of an object (GO statement) + if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then + # Determine object type and name + if echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then + object_type="tables" + object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/') + elif echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" >/dev/null; then + object_type="views" + object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" | sed -E 's/.*CREATE[[:space:]]+VIEW[[:space:]]+\[?([^\]]+)\]?.*/\1/') + elif echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC" >/dev/null; then + object_type="procedures" + object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC[[:space:]EDURE]*" | sed -E 's/.*CREATE[[:space:]]+PROC[[:space:]EDURE]*[[:space:]]+\[?([^\]]+)\]?.*/\1/') + elif echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" >/dev/null; then + # Extract table name and constraint name if it exists + table_name=$(echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" | sed -E 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/') + + if echo "$current_object" | grep -i "ADD[[:space:]]\+CONSTRAINT" >/dev/null; then + object_type="constraints" + object_name=$(echo "$current_object" | grep -i "CONSTRAINT" | sed -E 's/.*CONSTRAINT[[:space:]]+\[?([^\]]+)\]?.*/\1/') + else + object_type="alter_tables" + object_name="${table_name}_alter_$(date +%s%N | cut -b1-5)" + fi + elif echo "$current_object" | grep -i "CREATE.*INDEX" >/dev/null; then + object_type="indexes" + object_name=$(echo "$current_object" | grep -i "INDEX" | sed -E 's/.*INDEX[[:space:]]+\[?([^\]]+)\]?.*/\1/') + fi + + # If we identified an object type and name, save it + if [ -n "$object_type" ] && [ -n "$object_name" ]; then + # Clean the filename + clean_name=$(clean_filename "$object_name") + output_file="$BASE_DIR/$object_type/$clean_name.sql" + + # For alter_tables, make sure we don't overwrite existing files + if [ "$object_type" = "alter_tables" ]; then + counter=1 + base_file="$BASE_DIR/$object_type/$clean_name" + while [ -f "${base_file}.sql" ]; do + clean_name="${clean_name}_${counter}" + counter=$((counter + 1)) + done + output_file="$base_file.sql" + fi + + # Save the object to file + echo "$current_object" > "$output_file" + echo "Created: $output_file" + fi + + # Reset for next object + current_object="" + object_type="" + object_name="" + table_name="" + fi + done < "$SQL_FILE" +} + +# Remove comment lines starting with -- +echo "Removing comments and processing SQL file..." +sed 's/--.*$//' "$SQL_FILE" | extract_objects + +echo "SQL objects have been organized in the $BASE_DIR directory." +echo " +Directory structure: +$BASE_DIR/ +├── tables/ +├── views/ +├── procedures/ +├── constraints/ +├── indexes/ +└── alter_tables/" + +# Count files in each directory +echo -e "\nFile counts:" +for dir in tables views procedures constraints indexes alter_tables; do + count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l) + echo "$dir: $count files" +done \ No newline at end of file