include parsing files
This commit is contained in:
Ubuntu
parent
89b19e7f62
commit
9ae8ac1501
@ -15,6 +15,7 @@ services:
|
||||
- DROP_DATABASE=${DROP_DATABASE:-false}
|
||||
volumes:
|
||||
- sqlserver_data:/var/opt/mssql
|
||||
- ./logs:/usr/src/app/logs
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
|
||||
117
ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh
Executable file
117
ef-migration/infrastructure/sql-server/full_schemas/extract_constraints.sh
Executable file
@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if input file is provided
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <sql_file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SQL_FILE="$1"
|
||||
BASE_DIR="sql_objects"
|
||||
|
||||
# Create directory structure
|
||||
mkdir -p "$BASE_DIR"/{constraints,foreign_keys}
|
||||
|
||||
# Function to clean filename
|
||||
clean_filename() {
|
||||
# Remove brackets, convert dots and spaces to underscores, remove parentheses
|
||||
echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
|
||||
}
|
||||
|
||||
# Function to extract table name from constraint
|
||||
extract_table_name() {
|
||||
echo "$1" | sed -nE 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^]]+)\]?.*/\1/p'
|
||||
}
|
||||
|
||||
# Function to extract constraint name
|
||||
extract_constraint_name() {
|
||||
echo "$1" | sed -nE 's/.*CONSTRAINT[[:space:]]+\[?([^]]+)\]?.*/\1/p'
|
||||
}
|
||||
|
||||
# Function to process each ALTER TABLE statement
|
||||
process_statement() {
|
||||
local line="$1"
|
||||
local table_name=$(extract_table_name "$line")
|
||||
local constraint_name=$(extract_constraint_name "$line")
|
||||
|
||||
# Skip if we couldn't extract table or constraint name
|
||||
if [ -z "$table_name" ] || [ -z "$constraint_name" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Determine if it's a foreign key
|
||||
if echo "$line" | grep -i "FOREIGN[[:space:]]\+KEY" >/dev/null; then
|
||||
output_dir="$BASE_DIR/foreign_keys"
|
||||
else
|
||||
output_dir="$BASE_DIR/constraints"
|
||||
fi
|
||||
|
||||
# Clean filename and save
|
||||
clean_name=$(clean_filename "${table_name}_${constraint_name}")
|
||||
output_file="${output_dir}/${clean_name}.sql"
|
||||
echo "$line" > "$output_file"
|
||||
echo "Created: $output_file"
|
||||
}
|
||||
|
||||
# Main processing function
|
||||
extract_constraints() {
|
||||
local current_statement=""
|
||||
|
||||
# Read the file line by line
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
# Skip empty lines
|
||||
if [ -z "${line// }" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# If line starts with ALTER TABLE, it's a new statement
|
||||
if echo "$line" | grep -i "^[[:space:]]*ALTER[[:space:]]\+TABLE" >/dev/null; then
|
||||
current_statement="$line"
|
||||
|
||||
# If GO is on the same line
|
||||
if echo "$line" | grep -i "GO[[:space:]]*$" >/dev/null; then
|
||||
# Remove GO and process
|
||||
current_statement=$(echo "$current_statement" | sed 's/[[:space:]]*GO[[:space:]]*$//')
|
||||
process_statement "$current_statement"
|
||||
current_statement=""
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# If we find GO, process the current statement
|
||||
if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
|
||||
if [ -n "$current_statement" ]; then
|
||||
process_statement "$current_statement"
|
||||
current_statement=""
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Add line to current statement if we're collecting one
|
||||
if [ -n "$current_statement" ]; then
|
||||
current_statement+=" $line"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Remove comment lines starting with -- and process
|
||||
echo "Extracting constraints and foreign keys..."
|
||||
sed 's/--.*$//' "$SQL_FILE" | extract_constraints
|
||||
|
||||
echo "Constraints and foreign keys have been saved in $BASE_DIR/constraints and $BASE_DIR/foreign_keys"
|
||||
|
||||
# Count and list files
|
||||
echo -e "\nFile counts:"
|
||||
for dir in constraints foreign_keys; do
|
||||
count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l || echo 0)
|
||||
echo "$dir: $count files"
|
||||
|
||||
echo -e "\nExtracted ${dir}:"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
for file in "$BASE_DIR/$dir"/*.sql; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "- $(basename "$file")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
105
ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh
Executable file
105
ef-migration/infrastructure/sql-server/full_schemas/extract_tables.sh
Executable file
@ -0,0 +1,105 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if input file is provided
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <sql_file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SQL_FILE="$1"
|
||||
BASE_DIR="sql_objects"
|
||||
|
||||
# Create directory structure
|
||||
mkdir -p "$BASE_DIR/tables"
|
||||
|
||||
# Function to clean filename
|
||||
clean_filename() {
|
||||
# Remove brackets, convert dots and spaces to underscores, remove parentheses
|
||||
echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
|
||||
}
|
||||
|
||||
# Function to check if table is temporary
|
||||
is_temporary_table() {
|
||||
local table_name="$1"
|
||||
if echo "$table_name" | grep -iE "temp|tmp|temporary|import_|viewimport_" >/dev/null; then
|
||||
return 0 # true in bash
|
||||
else
|
||||
return 1 # false in bash
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to extract CREATE TABLE statements only
|
||||
extract_tables() {
|
||||
local current_object=""
|
||||
local capture_table=false
|
||||
local table_name=""
|
||||
|
||||
# Read the file line by line
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
# Check if we've found a CREATE TABLE statement
|
||||
if echo "$line" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then
|
||||
# Extract table name and check if it's temporary
|
||||
table_name=$(echo "$line" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
|
||||
if is_temporary_table "$table_name"; then
|
||||
echo "Skipping temporary table: $table_name"
|
||||
continue
|
||||
fi
|
||||
|
||||
capture_table=true
|
||||
current_object="$line"$'\n'
|
||||
continue
|
||||
fi
|
||||
|
||||
# If we're capturing a table and find a following CREATE or ALTER statement, stop capturing
|
||||
if $capture_table; then
|
||||
if echo "$line" | grep -i "^[[:space:]]*CREATE\|^[[:space:]]*ALTER" >/dev/null; then
|
||||
# Save the current table before starting a new object
|
||||
if [ -n "$table_name" ]; then
|
||||
clean_name=$(clean_filename "$table_name")
|
||||
output_file="$BASE_DIR/tables/${clean_name}.sql"
|
||||
echo "$current_object" > "$output_file"
|
||||
echo "Created: $output_file"
|
||||
fi
|
||||
capture_table=false
|
||||
current_object=""
|
||||
table_name=""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Add line to current object if we're still capturing
|
||||
current_object+="$line"$'\n'
|
||||
|
||||
# If we find a GO statement, save the table
|
||||
if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
|
||||
if [ -n "$table_name" ]; then
|
||||
clean_name=$(clean_filename "$table_name")
|
||||
output_file="$BASE_DIR/tables/${clean_name}.sql"
|
||||
echo "$current_object" > "$output_file"
|
||||
echo "Created: $output_file"
|
||||
fi
|
||||
capture_table=false
|
||||
current_object=""
|
||||
table_name=""
|
||||
fi
|
||||
fi
|
||||
done < "$SQL_FILE"
|
||||
}
|
||||
|
||||
# Remove comment lines starting with --
|
||||
echo "Extracting CREATE TABLE statements..."
|
||||
sed 's/--.*$//' "$SQL_FILE" | extract_tables
|
||||
|
||||
echo "CREATE TABLE statements have been saved in $BASE_DIR/tables/"
|
||||
|
||||
# Count files
|
||||
count=$(ls -1 "$BASE_DIR/tables"/*.sql 2>/dev/null | wc -l)
|
||||
echo -e "\nTotal tables extracted: $count"
|
||||
|
||||
# List extracted tables
|
||||
echo -e "\nExtracted tables:"
|
||||
for table in "$BASE_DIR/tables"/*.sql; do
|
||||
if [ -f "$table" ]; then
|
||||
echo "- $(basename "$table")"
|
||||
fi
|
||||
done
|
||||
124
ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh
Executable file
124
ef-migration/infrastructure/sql-server/full_schemas/extract_views.sh
Executable file
@ -0,0 +1,124 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if input file is provided
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <sql_file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SQL_FILE="$1"
|
||||
BASE_DIR="sql_objects"
|
||||
|
||||
# Create directory structure
|
||||
mkdir -p "$BASE_DIR/views"
|
||||
|
||||
# Function to clean filename
|
||||
clean_filename() {
|
||||
# Remove brackets, convert dots and spaces to underscores, remove parentheses
|
||||
echo "$1" | sed 's/\[\|\]//g' | tr '.' '_' | tr ' ' '_' | sed 's/[()]//g' | sed 's/__*/_/g' | sed 's/_$//'
|
||||
}
|
||||
|
||||
# Function to extract view name
|
||||
extract_view_name() {
|
||||
# More flexible pattern for extracting view name
|
||||
echo "$1" | sed -n 's/.*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]*[Vv][Ii][Ee][Ww][[:space:]]*\[\([^]]*\)\]\.\[\([^]]*\)\].*/\1.\2/p'
|
||||
}
|
||||
|
||||
# Function to clean the input line
|
||||
clean_input_line() {
|
||||
# Remove any leading/trailing whitespace and normalize spaces
|
||||
echo "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | tr -s ' '
|
||||
}
|
||||
|
||||
# Function to process each CREATE VIEW statement
|
||||
process_view() {
|
||||
local current_object="$1"
|
||||
local view_name="$2"
|
||||
|
||||
# Skip if we couldn't extract view name
|
||||
if [ -z "$view_name" ]; then
|
||||
echo "Warning: Could not extract view name from statement:"
|
||||
echo "$current_object" | head -n 1
|
||||
# Try to extract with a more lenient pattern
|
||||
view_name=$(echo "$current_object" | head -n 1 | grep -io '\[int\]\.\[[^]]*\]' | head -n 1 | sed 's/\[\|\]//g')
|
||||
if [ -z "$view_name" ]; then
|
||||
return
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clean filename and save
|
||||
clean_name=$(clean_filename "$view_name")
|
||||
output_file="$BASE_DIR/views/${clean_name}.sql"
|
||||
echo "$current_object" > "$output_file"
|
||||
echo "Created: $output_file (from view: $view_name)"
|
||||
}
|
||||
|
||||
# Main processing function
|
||||
extract_views() {
|
||||
local current_object=""
|
||||
local capture_view=false
|
||||
local view_name=""
|
||||
|
||||
# Read the file line by line
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
# Clean the line
|
||||
line=$(clean_input_line "$line")
|
||||
|
||||
# Skip empty lines at the start of an object
|
||||
if [ -z "$line" ] && [ -z "$current_object" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check if we've found a CREATE VIEW statement (case insensitive and flexible)
|
||||
if echo "$line" | grep -iE "[[:space:]]*[Cc][Rr][Ee][Aa][Tt][Ee][[:space:]]+[Vv][Ii][Ee][Ww]" >/dev/null; then
|
||||
capture_view=true
|
||||
current_object="$line"
|
||||
view_name=$(extract_view_name "$line")
|
||||
if [ -n "$view_name" ]; then
|
||||
echo "Found view: $view_name"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# If we're capturing a view
|
||||
if $capture_view; then
|
||||
# Check if we've reached the end (GO statement)
|
||||
if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
|
||||
# Add GO to the view definition
|
||||
current_object+=$'\n'"GO"
|
||||
process_view "$current_object" "$view_name"
|
||||
capture_view=false
|
||||
current_object=""
|
||||
view_name=""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Add line to current object
|
||||
current_object+=$'\n'"$line"
|
||||
fi
|
||||
done
|
||||
|
||||
# Handle case where last view might not have a GO statement
|
||||
if $capture_view && [ -n "$current_object" ]; then
|
||||
process_view "$current_object" "$view_name"
|
||||
fi
|
||||
}
|
||||
|
||||
# Pre-process the file to normalize line endings and remove comments
|
||||
echo "Extracting views..."
|
||||
sed 's/--.*$//; /^[[:space:]]*$/d' "$SQL_FILE" | extract_views
|
||||
|
||||
echo "Views have been saved in $BASE_DIR/views"
|
||||
|
||||
# Count and list files
|
||||
count=$(ls -1 "$BASE_DIR/views"/*.sql 2>/dev/null | wc -l || echo 0)
|
||||
echo -e "\nTotal views extracted: $count"
|
||||
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo -e "\nExtracted views:"
|
||||
for file in "$BASE_DIR/views"/*.sql; do
|
||||
if [ -f "$file" ]; then
|
||||
echo "- $(basename "$file")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
116
ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh
Executable file
116
ef-migration/infrastructure/sql-server/full_schemas/parse_sql.sh
Executable file
@ -0,0 +1,116 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check if input file is provided
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <sql_file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SQL_FILE="$1"
|
||||
BASE_DIR="sql_objects"
|
||||
|
||||
# Create directory structure
|
||||
mkdir -p "$BASE_DIR"/{tables,views,procedures,constraints,indexes,alter_tables}
|
||||
|
||||
# Function to clean filename
|
||||
clean_filename() {
|
||||
echo "$1" | sed 's/\[\|\]//g' | tr '.' '_'
|
||||
}
|
||||
|
||||
# Function to extract SQL objects and save to files
|
||||
extract_objects() {
|
||||
local current_object=""
|
||||
local object_type=""
|
||||
local object_name=""
|
||||
local table_name=""
|
||||
|
||||
# Read the file line by line
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
# Skip empty lines at the start
|
||||
if [ -z "$current_object" ] && [ -z "${line// }" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Add line to current object
|
||||
current_object+="$line"$'\n'
|
||||
|
||||
# Check if we've reached the end of an object (GO statement)
|
||||
if echo "$line" | grep -i "^[[:space:]]*GO[[:space:]]*$" >/dev/null; then
|
||||
# Determine object type and name
|
||||
if echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" >/dev/null; then
|
||||
object_type="tables"
|
||||
object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+TABLE" | sed -E 's/.*CREATE[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
elif echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" >/dev/null; then
|
||||
object_type="views"
|
||||
object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+VIEW" | sed -E 's/.*CREATE[[:space:]]+VIEW[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
elif echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC" >/dev/null; then
|
||||
object_type="procedures"
|
||||
object_name=$(echo "$current_object" | grep -i "CREATE[[:space:]]\+PROC[[:space:]EDURE]*" | sed -E 's/.*CREATE[[:space:]]+PROC[[:space:]EDURE]*[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
elif echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" >/dev/null; then
|
||||
# Extract table name and constraint name if it exists
|
||||
table_name=$(echo "$current_object" | grep -i "ALTER[[:space:]]\+TABLE" | sed -E 's/.*ALTER[[:space:]]+TABLE[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
|
||||
if echo "$current_object" | grep -i "ADD[[:space:]]\+CONSTRAINT" >/dev/null; then
|
||||
object_type="constraints"
|
||||
object_name=$(echo "$current_object" | grep -i "CONSTRAINT" | sed -E 's/.*CONSTRAINT[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
else
|
||||
object_type="alter_tables"
|
||||
object_name="${table_name}_alter_$(date +%s%N | cut -b1-5)"
|
||||
fi
|
||||
elif echo "$current_object" | grep -i "CREATE.*INDEX" >/dev/null; then
|
||||
object_type="indexes"
|
||||
object_name=$(echo "$current_object" | grep -i "INDEX" | sed -E 's/.*INDEX[[:space:]]+\[?([^\]]+)\]?.*/\1/')
|
||||
fi
|
||||
|
||||
# If we identified an object type and name, save it
|
||||
if [ -n "$object_type" ] && [ -n "$object_name" ]; then
|
||||
# Clean the filename
|
||||
clean_name=$(clean_filename "$object_name")
|
||||
output_file="$BASE_DIR/$object_type/$clean_name.sql"
|
||||
|
||||
# For alter_tables, make sure we don't overwrite existing files
|
||||
if [ "$object_type" = "alter_tables" ]; then
|
||||
counter=1
|
||||
base_file="$BASE_DIR/$object_type/$clean_name"
|
||||
while [ -f "${base_file}.sql" ]; do
|
||||
clean_name="${clean_name}_${counter}"
|
||||
counter=$((counter + 1))
|
||||
done
|
||||
output_file="$base_file.sql"
|
||||
fi
|
||||
|
||||
# Save the object to file
|
||||
echo "$current_object" > "$output_file"
|
||||
echo "Created: $output_file"
|
||||
fi
|
||||
|
||||
# Reset for next object
|
||||
current_object=""
|
||||
object_type=""
|
||||
object_name=""
|
||||
table_name=""
|
||||
fi
|
||||
done < "$SQL_FILE"
|
||||
}
|
||||
|
||||
# Remove comment lines starting with --
|
||||
echo "Removing comments and processing SQL file..."
|
||||
sed 's/--.*$//' "$SQL_FILE" | extract_objects
|
||||
|
||||
echo "SQL objects have been organized in the $BASE_DIR directory."
|
||||
echo "
|
||||
Directory structure:
|
||||
$BASE_DIR/
|
||||
├── tables/
|
||||
├── views/
|
||||
├── procedures/
|
||||
├── constraints/
|
||||
├── indexes/
|
||||
└── alter_tables/"
|
||||
|
||||
# Count files in each directory
|
||||
echo -e "\nFile counts:"
|
||||
for dir in tables views procedures constraints indexes alter_tables; do
|
||||
count=$(ls -1 "$BASE_DIR/$dir"/*.sql 2>/dev/null | wc -l)
|
||||
echo "$dir: $count files"
|
||||
done
|
||||
Loading…
Reference in New Issue
Block a user