diff --git a/DBRepair.sh b/DBRepair.sh index adcfbb2..3e8905b 100755 --- a/DBRepair.sh +++ b/DBRepair.sh @@ -62,6 +62,17 @@ STATPERMS="%a" # On all hosts except QNAP DFFLAGS="-m" +# FTS4 virtual table query - excludes shadow tables (_content, _segments, etc.) +FTS_TABLE_QUERY="SELECT name FROM sqlite_master" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY WHERE type='table'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND sql LIKE '%fts4%'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_content'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_segments'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_segdir'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_stat'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_docsize'" +FTS_TABLE_QUERY="$FTS_TABLE_QUERY ORDER BY name;" + # If LC_ALL is null, default to C [ "$LC_ALL" = "" ] && export LC_ALL=C @@ -158,6 +169,83 @@ CheckDatabases() { return $Damaged } +# Global flags for FTS status (separate from main DB) +FTSDamaged=0 +CheckedFTS=0 + +# Check FTS (Full-Text Search) index integrity +CheckFTS() { + + # Arg1 = calling function name for logging + + FTSDamaged=0 + local Caller="${1:-Check}" + local FTSFail=0 + + Output "Checking FTS (Full-Text Search) indexes" + + # Get list of FTS4 virtual tables (exclude shadow tables) + FTSTables="$("$PLEX_SQLITE" $CPPL.db "$FTS_TABLE_QUERY" 2>&1)" + + if [ -z "$FTSTables" ]; then + Output "No FTS4 tables found in main database." + WriteLog "$Caller - FTS Check - No FTS4 tables" + return 0 + fi + + # Check each FTS table + for Table in $FTSTables + do + Result="$("$PLEX_SQLITE" $CPPL.db "INSERT INTO $Table($Table) VALUES('integrity-check');" 2>&1)" + ExitCode=$? + + if [ $ExitCode -eq 0 ] && [ -z "$Result" ]; then + Output " FTS index '$Table' - OK" + WriteLog "$Caller - FTS Check: $Table - PASS" + else + Output " FTS index '$Table' - DAMAGED" + Output " Error: $Result" + WriteLog "$Caller - FTS Check: $Table - FAIL ($Result)" + FTSFail=1 + fi + done + + # Check blobs database FTS tables + FTSTablesBlobs="$("$PLEX_SQLITE" $CPPL.blobs.db "$FTS_TABLE_QUERY" 2>&1)" + + if [ -n "$FTSTablesBlobs" ]; then + for Table in $FTSTablesBlobs + do + Result="$("$PLEX_SQLITE" $CPPL.blobs.db "INSERT INTO $Table($Table) VALUES('integrity-check');" 2>&1)" + ExitCode=$? + + if [ $ExitCode -eq 0 ] && [ -z "$Result" ]; then + Output " FTS index '$Table' (blobs) - OK" + WriteLog "$Caller - FTS Check (blobs): $Table - PASS" + else + Output " FTS index '$Table' (blobs) - DAMAGED" + Output " Error: $Result" + WriteLog "$Caller - FTS Check (blobs): $Table - FAIL ($Result)" + FTSFail=1 + fi + done + fi + + CheckedFTS=1 + if [ $FTSFail -eq 0 ]; then + Output "FTS integrity check complete. All FTS indexes OK." + WriteLog "$Caller - FTS Check - PASS" + FTSDamaged=0 + else + Output "FTS integrity check complete. One or more FTS indexes are DAMAGED." + Output "Use 'reindex' command (option 6) or 'automatic' (option 2) to rebuild." + WriteLog "$Caller - FTS Check - FAIL" + FTSDamaged=1 + fi + + return $FTSFail +} + # Return list of database backup dates for consideration in replace action GetDates(){ @@ -932,6 +1020,151 @@ DoIndex() { } +##### FTS REBUILD +DoFTSRebuild() { + + # Clear flags + Damaged=0 + Fail=0 + + # Check databases before rebuilding FTS if not previously checked + if ! CheckDatabases "FTSRbld" ; then + Damaged=1 + CheckedDB=1 + Fail=1 + [ $IgnoreErrors -eq 1 ] && Fail=0 + fi + + # If damaged, warn but allow continue (FTS corruption often passes integrity_check) + if [ $Damaged -eq 1 ] && [ $IgnoreErrors -eq 0 ]; then + Output "WARNING: Database integrity check failed." + Output "FTS rebuild may still help if the corruption is isolated to FTS indexes." + if ! ConfirmYesNo "Continue with FTS rebuild anyway? "; then + Output "FTS rebuild cancelled." + return 1 + fi + fi + + # Make backup + Output "Backing up databases" + MakeBackups "FTSRbld" + Result=$? + [ $IgnoreErrors -eq 1 ] && Result=0 + + if [ $Result -eq 0 ]; then + WriteLog "FTSRbld - MakeBackup - PASS" + else + Output "Error making backups. Cannot continue." + WriteLog "FTSRbld - MakeBackup - FAIL ($Result)" + Fail=1 + return 1 + fi + + # Get list of FTS4 tables (exclude shadow tables) + Output "Scanning for FTS4 tables in main database..." + + FTSTables="$("$PLEX_SQLITE" $CPPL.db "$FTS_TABLE_QUERY" 2>&1)" + Result=$? + + if ! SQLiteOK $Result; then + Output "Error scanning for FTS tables. Error code $Result" + WriteLog "FTSRbld - Scan FTS tables - FAIL ($Result)" + Fail=1 + RestoreSaved "$TimeStamp" + return 1 + fi + + # If no FTS tables found, nothing to do + if [ -z "$FTSTables" ]; then + Output "No FTS4 tables found in database." + WriteLog "FTSRbld - No FTS4 tables found" + return 0 + fi + + Output "Found FTS4 tables:" + for Table in $FTSTables + do + Output " - $Table" + done + Output "" + + # Rebuild each FTS table + Output "Rebuilding FTS4 indexes in main database..." + for Table in $FTSTables + do + Output " Rebuilding $Table..." + + "$PLEX_SQLITE" $CPPL.db "INSERT INTO $Table($Table) VALUES('rebuild');" 2>&1 + Result=$? + [ $IgnoreErrors -eq 1 ] && Result=0 + + if SQLiteOK $Result; then + Output " $Table rebuilt successfully." + WriteLog "FTSRbld - Rebuild: $Table - PASS" + else + Output " $Table rebuild failed. Error code $Result" + WriteLog "FTSRbld - Rebuild: $Table - FAIL ($Result)" + Fail=1 + fi + done + + # Check blobs database for FTS tables + Output "" + Output "Scanning for FTS4 tables in blobs database..." + + FTSTablesBlobs="$("$PLEX_SQLITE" $CPPL.blobs.db "$FTS_TABLE_QUERY" 2>&1)" + Result=$? + + if ! SQLiteOK $Result; then + Output "Error scanning blobs database for FTS tables. Error code $Result" + WriteLog "FTSRbld - Scan FTS tables (blobs) - FAIL ($Result)" + # Don't fail entirely if blobs scan fails - main DB may be OK + elif [ -z "$FTSTablesBlobs" ]; then + Output "No FTS4 tables found in blobs database." + WriteLog "FTSRbld - No FTS4 tables in blobs database" + else + Output "Found FTS4 tables in blobs database:" + for Table in $FTSTablesBlobs + do + Output " - $Table" + done + Output "" + + Output "Rebuilding FTS4 indexes in blobs database..." + for Table in $FTSTablesBlobs + do + Output " Rebuilding $Table..." + + "$PLEX_SQLITE" $CPPL.blobs.db "INSERT INTO $Table($Table) VALUES('rebuild');" 2>&1 + Result=$? + [ $IgnoreErrors -eq 1 ] && Result=0 + + if SQLiteOK $Result; then + Output " $Table rebuilt successfully." + WriteLog "FTSRbld - Rebuild (blobs): $Table - PASS" + else + Output " $Table rebuild failed. Error code $Result" + WriteLog "FTSRbld - Rebuild (blobs): $Table - FAIL ($Result)" + Fail=1 + fi + done + fi + + Output "" + Output "FTS rebuild complete." + + if [ $Fail -eq 0 ]; then + SetLast "FTSRbld" "$TimeStamp" + WriteLog "FTSRbld - PASS" + else + Output "Some FTS tables failed to rebuild. Restoring backup." + RestoreSaved "$TimeStamp" + WriteLog "FTSRbld - FAIL" + fi + + return $Fail +} + ##### UNDO DoUndo(){ # Confirm there is something to undo @@ -2188,8 +2421,8 @@ do echo "" [ $HaveStartStop -gt 0 ] && echo " 1 - 'stop' - Stop PMS." [ $HaveStartStop -eq 0 ] && echo " 1 - 'stop' - (Not available. Stop manually.)" - echo " 2 - 'automatic' - Check, Repair/Optimize, and Reindex Database in one step." - echo " 3 - 'check' - Perform integrity check of database." + echo " 2 - 'automatic' - Check, Repair/Optimize, Reindex, and FTS rebuild in one step." + echo " 3 - 'check' - Perform integrity check of database and FTS indexes." echo " 4 - 'vacuum' - Remove empty space from database without optimizing." echo " 5 - 'repair' - Repair/Optimize databases." echo " 6 - 'reindex' - Rebuild database indexes." @@ -2326,9 +2559,29 @@ do WriteLog "Reindex - PASS" fi + # Now check FTS indexes and repair if damaged + DoUpdateTimestamp + Output "" + if ! CheckFTS "Auto "; then + Output "" + Output "FTS indexes are damaged. Attempting automatic FTS rebuild..." + WriteLog "Auto - FTS damaged, attempting rebuild" + + if DoFTSRebuild; then + WriteLog "Auto - FTS Rebuild - PASS" + Output "FTS rebuild successful." + else + WriteLog "Auto - FTS Rebuild - FAIL" + Output "FTS rebuild failed. You may need to run 'reindex' command manually." + # Don't fail auto entirely - main DB repair succeeded + fi + else + WriteLog "Auto - FTS Check - PASS" + fi + # All good to here WriteLog "Auto - COMPLETED" - Output "Automatic Check, Repair/optimize, & Index successful." + Output "Automatic Check, Repair/optimize, Index, & FTS check successful." ;; @@ -2350,6 +2603,14 @@ do WriteLog "Check - FAIL" CheckedDB=0 fi + + # Also check FTS indexes (these can be damaged even when integrity_check passes) + Output "" + if ! CheckFTS "Check "; then + Output "" + Output "NOTE: FTS indexes are damaged but main database structure is OK." + Output " Use 'reindex' (option 6) or 'automatic' (option 2) to rebuild." + fi ;; # Vacuum @@ -2413,6 +2674,24 @@ do # Now index if DoIndex ; then WriteLog "Reindex - PASS" + + # Check FTS indexes and repair if damaged + Output "" + if ! CheckFTS "Reindex"; then + Output "" + Output "FTS indexes are damaged. Rebuilding..." + WriteLog "Reindex - FTS damaged, attempting rebuild" + + if DoFTSRebuild; then + WriteLog "Reindex - FTS Rebuild - PASS" + Output "FTS rebuild successful." + else + WriteLog "Reindex - FTS Rebuild - FAIL" + Output "FTS rebuild failed." + fi + else + WriteLog "Reindex - FTS Check - PASS" + fi else WriteLog "Reindex - FAIL" fi @@ -2494,6 +2773,9 @@ do [ $CheckedDB -eq 0 ] && Output " Databases are not checked, Status unknown." [ $CheckedDB -eq 1 ] && [ $Damaged -eq 0 ] && Output " Databases are OK." [ $CheckedDB -eq 1 ] && [ $Damaged -eq 1 ] && Output " Databases were checked and are damaged." + [ $CheckedFTS -eq 0 ] && Output " FTS indexes are not checked, Status unknown." + [ $CheckedFTS -eq 1 ] && [ $FTSDamaged -eq 0 ] && Output " FTS indexes are OK." + [ $CheckedFTS -eq 1 ] && [ $FTSDamaged -eq 1 ] && Output " FTS indexes are damaged." Output "" ;; @@ -2713,4 +2995,4 @@ do esac done done -exit 0 \ No newline at end of file +exit 0 diff --git a/README.md b/README.md index 2b15bd2..602eac3 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,15 @@ If sufficient privleges exist (root), and supported by the environment, the opti The following commands (or their number), listed in alphabetical order, are accepted as input. ``` - AUTO(matic) - Automatically check, repair/optimize, and reindex the databases in one step. - CHEC(k) - Check the main and blob databases integrity + AUTO(matic) - Automatically check, repair/optimize, reindex, and FTS rebuild in one step. + CHEC(k) - Check the main and blob databases integrity (includes FTS index check) DEFL(ate) - Deflate a bloated PMS database (faulty statistics data) EXIT - Exit the utility IGNOre/HONOr - Ignore/Honor constraint errors when IMPORTing additional data into DB. IMPO(rt) - Import viewstate / watch history from another database PRUN(e) - Prune (remove) old image files from transcoder cache diretory PURG(e) - Purge (delete) all temporary files left behind by PMS & the transcoder from the temp directory - REIN(dex) - Rebuild the database indexes + REIN(dex) - Rebuild the database indexes (includes FTS indexes) REPL(ace) - Replace the existing databases with a PMS-generated backup SHOW - Show the log file STAR(t) - Start PMS (not available on all platforms) @@ -58,8 +58,8 @@ If sufficient privleges exist (root), and supported by the environment, the opti Select 1 - 'stop' - Stop PMS. - 2 - 'automatic' - Check, Repair/Optimize, and Reindex Database in one step. - 3 - 'check' - Perform integrity check of database. + 2 - 'automatic' - Check, Repair/Optimize, Reindex, and FTS rebuild in one step. + 3 - 'check' - Perform integrity check of database and FTS indexes. 4 - 'vacuum' - Remove empty space from database without optimizing. 5 - 'repair' - Repair/Optimize databases. 6 - 'reindex' - Rebuild database indexes. @@ -105,7 +105,7 @@ Enter command # -or- command name (4 char min) : # Installation ### Downloading - Download DBRepair.sh (if you want just the script) + Download DBRepair.sh (if you want just the script) # This overwrites any existing version. Remove "-O DBRepair.sh" to not overwrite. ``` wget -O DBRepair.sh https://github.com/ChuckPa/PlexDBRepair/releases/latest/download/DBRepair.sh @@ -147,10 +147,10 @@ Enter command # -or- command name (4 char min) : 1. Accessing a NAS Open a terminal/command line window on your computer. type: ssh admin-username@IP.addr.of.NAS - - 2. Linux + + 2. Linux Open a terminal session and elevate to the root (sudo) user - + 3. Windows -- for Windows PMS hosts Open a Command window Follow the instructions for the Windows version of DBRepair @@ -265,6 +265,17 @@ These examples than you were before. In this case, UNDO then Repair. Undo can only undo the single most-recent action. (Note: In a future release, you will be able to 'undo' every action taken until the DBs are in their original state) + G. HTTP 500 errors when adding to collections / FTS index corruption + This occurs when standard integrity_check passes but FTS (Full-Text Search) indexes are corrupted. + Symptoms: Adding items to collections fails, updating metadata fails, "database disk image is malformed" + during UPDATE operations even though Check reports databases are OK. + + 1. (3) Check - Will show "FTS index damaged" message + 2. (6) Reindex - Rebuild indexes including FTS + 3. (99) Exit + + Alternatively, use (2) Automatic which will detect and repair FTS issues automatically. + Special considerations: 1. As stated above, this utility requires PMS to be stopped in order to do what it does. @@ -667,6 +678,13 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases# Checks the integrity of the Plex main and blobs databases. + Also performs FTS (Full-Text Search) index integrity checks. FTS indexes can become + corrupted even when standard integrity checks pass, causing operations like adding + items to collections to fail with "database disk image is malformed" errors. + + If FTS corruption is detected, use 'reindex' (option 6) or 'automatic' (option 2) + to rebuild the FTS indexes. + ### Deflate Repairs a known error in the PMS main database "statistics_bandwidth" table. @@ -710,6 +728,11 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases# Rebuilds the database indexes after an import, repair, or replace operation. These indexes are used by PMS for searching (both internally and your typed searches) + Also checks FTS (Full-Text Search) index integrity and rebuilds if corruption is detected. + FTS indexes can become corrupted even when standard integrity checks pass, causing + "database disk image is malformed" errors during UPDATE operations (e.g., adding items + to collections, updating metadata). + ### Repair Extracts/recovers all the usable data from the existing databases into text (SQL ascii) form.