mirror of
https://github.com/ChuckPa/PlexDBRepair.git
synced 2026-02-04 06:22:59 -05:00
feat: Add FTS4 index integrity check and rebuild functionality
Add detection and automatic repair for FTS4 (Full-Text Search) index corruption, which can occur even when PRAGMA integrity_check passes. New functions: - CheckFTS(): Validates FTS4 index integrity using SQLite's integrity-check command - DoFTSRebuild(): Rebuilds corrupted FTS4 indexes using SQLite's rebuild command Enhanced existing commands: - 'check' (option 3): Now includes FTS integrity checking - 'reindex' (option 6): Now checks FTS indexes and rebuilds if corruption detected - 'automatic' (option 2): Checks FTS after repair/reindex and automatically rebuilds if damaged Refactored FTS table query into reusable FTS_TABLE_QUERY constant for maintainability. Fixes: https://github.com/ChuckPa/DBRepair/issues/269
This commit is contained in:
Max Allison
288
DBRepair.sh
288
DBRepair.sh
@@ -62,6 +62,17 @@ STATPERMS="%a"
|
||||
# On all hosts except QNAP
|
||||
DFFLAGS="-m"
|
||||
|
||||
# FTS4 virtual table query - excludes shadow tables (_content, _segments, etc.)
|
||||
FTS_TABLE_QUERY="SELECT name FROM sqlite_master"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY WHERE type='table'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND sql LIKE '%fts4%'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_content'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_segments'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_segdir'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_stat'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY AND name NOT LIKE '%_docsize'"
|
||||
FTS_TABLE_QUERY="$FTS_TABLE_QUERY ORDER BY name;"
|
||||
|
||||
# If LC_ALL is null, default to C
|
||||
[ "$LC_ALL" = "" ] && export LC_ALL=C
|
||||
|
||||
@@ -158,6 +169,83 @@ CheckDatabases() {
|
||||
return $Damaged
|
||||
}
|
||||
|
||||
# Global flags for FTS status (separate from main DB)
|
||||
FTSDamaged=0
|
||||
CheckedFTS=0
|
||||
|
||||
# Check FTS (Full-Text Search) index integrity
|
||||
CheckFTS() {
|
||||
|
||||
# Arg1 = calling function name for logging
|
||||
|
||||
FTSDamaged=0
|
||||
local Caller="${1:-Check}"
|
||||
local FTSFail=0
|
||||
|
||||
Output "Checking FTS (Full-Text Search) indexes"
|
||||
|
||||
# Get list of FTS4 virtual tables (exclude shadow tables)
|
||||
FTSTables="$("$PLEX_SQLITE" $CPPL.db "$FTS_TABLE_QUERY" 2>&1)"
|
||||
|
||||
if [ -z "$FTSTables" ]; then
|
||||
Output "No FTS4 tables found in main database."
|
||||
WriteLog "$Caller - FTS Check - No FTS4 tables"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check each FTS table
|
||||
for Table in $FTSTables
|
||||
do
|
||||
Result="$("$PLEX_SQLITE" $CPPL.db "INSERT INTO $Table($Table) VALUES('integrity-check');" 2>&1)"
|
||||
ExitCode=$?
|
||||
|
||||
if [ $ExitCode -eq 0 ] && [ -z "$Result" ]; then
|
||||
Output " FTS index '$Table' - OK"
|
||||
WriteLog "$Caller - FTS Check: $Table - PASS"
|
||||
else
|
||||
Output " FTS index '$Table' - DAMAGED"
|
||||
Output " Error: $Result"
|
||||
WriteLog "$Caller - FTS Check: $Table - FAIL ($Result)"
|
||||
FTSFail=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check blobs database FTS tables
|
||||
FTSTablesBlobs="$("$PLEX_SQLITE" $CPPL.blobs.db "$FTS_TABLE_QUERY" 2>&1)"
|
||||
|
||||
if [ -n "$FTSTablesBlobs" ]; then
|
||||
for Table in $FTSTablesBlobs
|
||||
do
|
||||
Result="$("$PLEX_SQLITE" $CPPL.blobs.db "INSERT INTO $Table($Table) VALUES('integrity-check');" 2>&1)"
|
||||
ExitCode=$?
|
||||
|
||||
if [ $ExitCode -eq 0 ] && [ -z "$Result" ]; then
|
||||
Output " FTS index '$Table' (blobs) - OK"
|
||||
WriteLog "$Caller - FTS Check (blobs): $Table - PASS"
|
||||
else
|
||||
Output " FTS index '$Table' (blobs) - DAMAGED"
|
||||
Output " Error: $Result"
|
||||
WriteLog "$Caller - FTS Check (blobs): $Table - FAIL ($Result)"
|
||||
FTSFail=1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
CheckedFTS=1
|
||||
if [ $FTSFail -eq 0 ]; then
|
||||
Output "FTS integrity check complete. All FTS indexes OK."
|
||||
WriteLog "$Caller - FTS Check - PASS"
|
||||
FTSDamaged=0
|
||||
else
|
||||
Output "FTS integrity check complete. One or more FTS indexes are DAMAGED."
|
||||
Output "Use 'reindex' command (option 6) or 'automatic' (option 2) to rebuild."
|
||||
WriteLog "$Caller - FTS Check - FAIL"
|
||||
FTSDamaged=1
|
||||
fi
|
||||
|
||||
return $FTSFail
|
||||
}
|
||||
|
||||
# Return list of database backup dates for consideration in replace action
|
||||
GetDates(){
|
||||
|
||||
@@ -932,6 +1020,151 @@ DoIndex() {
|
||||
|
||||
}
|
||||
|
||||
##### FTS REBUILD
|
||||
DoFTSRebuild() {
|
||||
|
||||
# Clear flags
|
||||
Damaged=0
|
||||
Fail=0
|
||||
|
||||
# Check databases before rebuilding FTS if not previously checked
|
||||
if ! CheckDatabases "FTSRbld" ; then
|
||||
Damaged=1
|
||||
CheckedDB=1
|
||||
Fail=1
|
||||
[ $IgnoreErrors -eq 1 ] && Fail=0
|
||||
fi
|
||||
|
||||
# If damaged, warn but allow continue (FTS corruption often passes integrity_check)
|
||||
if [ $Damaged -eq 1 ] && [ $IgnoreErrors -eq 0 ]; then
|
||||
Output "WARNING: Database integrity check failed."
|
||||
Output "FTS rebuild may still help if the corruption is isolated to FTS indexes."
|
||||
if ! ConfirmYesNo "Continue with FTS rebuild anyway? "; then
|
||||
Output "FTS rebuild cancelled."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Make backup
|
||||
Output "Backing up databases"
|
||||
MakeBackups "FTSRbld"
|
||||
Result=$?
|
||||
[ $IgnoreErrors -eq 1 ] && Result=0
|
||||
|
||||
if [ $Result -eq 0 ]; then
|
||||
WriteLog "FTSRbld - MakeBackup - PASS"
|
||||
else
|
||||
Output "Error making backups. Cannot continue."
|
||||
WriteLog "FTSRbld - MakeBackup - FAIL ($Result)"
|
||||
Fail=1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get list of FTS4 tables (exclude shadow tables)
|
||||
Output "Scanning for FTS4 tables in main database..."
|
||||
|
||||
FTSTables="$("$PLEX_SQLITE" $CPPL.db "$FTS_TABLE_QUERY" 2>&1)"
|
||||
Result=$?
|
||||
|
||||
if ! SQLiteOK $Result; then
|
||||
Output "Error scanning for FTS tables. Error code $Result"
|
||||
WriteLog "FTSRbld - Scan FTS tables - FAIL ($Result)"
|
||||
Fail=1
|
||||
RestoreSaved "$TimeStamp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# If no FTS tables found, nothing to do
|
||||
if [ -z "$FTSTables" ]; then
|
||||
Output "No FTS4 tables found in database."
|
||||
WriteLog "FTSRbld - No FTS4 tables found"
|
||||
return 0
|
||||
fi
|
||||
|
||||
Output "Found FTS4 tables:"
|
||||
for Table in $FTSTables
|
||||
do
|
||||
Output " - $Table"
|
||||
done
|
||||
Output ""
|
||||
|
||||
# Rebuild each FTS table
|
||||
Output "Rebuilding FTS4 indexes in main database..."
|
||||
for Table in $FTSTables
|
||||
do
|
||||
Output " Rebuilding $Table..."
|
||||
|
||||
"$PLEX_SQLITE" $CPPL.db "INSERT INTO $Table($Table) VALUES('rebuild');" 2>&1
|
||||
Result=$?
|
||||
[ $IgnoreErrors -eq 1 ] && Result=0
|
||||
|
||||
if SQLiteOK $Result; then
|
||||
Output " $Table rebuilt successfully."
|
||||
WriteLog "FTSRbld - Rebuild: $Table - PASS"
|
||||
else
|
||||
Output " $Table rebuild failed. Error code $Result"
|
||||
WriteLog "FTSRbld - Rebuild: $Table - FAIL ($Result)"
|
||||
Fail=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check blobs database for FTS tables
|
||||
Output ""
|
||||
Output "Scanning for FTS4 tables in blobs database..."
|
||||
|
||||
FTSTablesBlobs="$("$PLEX_SQLITE" $CPPL.blobs.db "$FTS_TABLE_QUERY" 2>&1)"
|
||||
Result=$?
|
||||
|
||||
if ! SQLiteOK $Result; then
|
||||
Output "Error scanning blobs database for FTS tables. Error code $Result"
|
||||
WriteLog "FTSRbld - Scan FTS tables (blobs) - FAIL ($Result)"
|
||||
# Don't fail entirely if blobs scan fails - main DB may be OK
|
||||
elif [ -z "$FTSTablesBlobs" ]; then
|
||||
Output "No FTS4 tables found in blobs database."
|
||||
WriteLog "FTSRbld - No FTS4 tables in blobs database"
|
||||
else
|
||||
Output "Found FTS4 tables in blobs database:"
|
||||
for Table in $FTSTablesBlobs
|
||||
do
|
||||
Output " - $Table"
|
||||
done
|
||||
Output ""
|
||||
|
||||
Output "Rebuilding FTS4 indexes in blobs database..."
|
||||
for Table in $FTSTablesBlobs
|
||||
do
|
||||
Output " Rebuilding $Table..."
|
||||
|
||||
"$PLEX_SQLITE" $CPPL.blobs.db "INSERT INTO $Table($Table) VALUES('rebuild');" 2>&1
|
||||
Result=$?
|
||||
[ $IgnoreErrors -eq 1 ] && Result=0
|
||||
|
||||
if SQLiteOK $Result; then
|
||||
Output " $Table rebuilt successfully."
|
||||
WriteLog "FTSRbld - Rebuild (blobs): $Table - PASS"
|
||||
else
|
||||
Output " $Table rebuild failed. Error code $Result"
|
||||
WriteLog "FTSRbld - Rebuild (blobs): $Table - FAIL ($Result)"
|
||||
Fail=1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
Output ""
|
||||
Output "FTS rebuild complete."
|
||||
|
||||
if [ $Fail -eq 0 ]; then
|
||||
SetLast "FTSRbld" "$TimeStamp"
|
||||
WriteLog "FTSRbld - PASS"
|
||||
else
|
||||
Output "Some FTS tables failed to rebuild. Restoring backup."
|
||||
RestoreSaved "$TimeStamp"
|
||||
WriteLog "FTSRbld - FAIL"
|
||||
fi
|
||||
|
||||
return $Fail
|
||||
}
|
||||
|
||||
##### UNDO
|
||||
DoUndo(){
|
||||
# Confirm there is something to undo
|
||||
@@ -2188,8 +2421,8 @@ do
|
||||
echo ""
|
||||
[ $HaveStartStop -gt 0 ] && echo " 1 - 'stop' - Stop PMS."
|
||||
[ $HaveStartStop -eq 0 ] && echo " 1 - 'stop' - (Not available. Stop manually.)"
|
||||
echo " 2 - 'automatic' - Check, Repair/Optimize, and Reindex Database in one step."
|
||||
echo " 3 - 'check' - Perform integrity check of database."
|
||||
echo " 2 - 'automatic' - Check, Repair/Optimize, Reindex, and FTS rebuild in one step."
|
||||
echo " 3 - 'check' - Perform integrity check of database and FTS indexes."
|
||||
echo " 4 - 'vacuum' - Remove empty space from database without optimizing."
|
||||
echo " 5 - 'repair' - Repair/Optimize databases."
|
||||
echo " 6 - 'reindex' - Rebuild database indexes."
|
||||
@@ -2326,9 +2559,29 @@ do
|
||||
WriteLog "Reindex - PASS"
|
||||
fi
|
||||
|
||||
# Now check FTS indexes and repair if damaged
|
||||
DoUpdateTimestamp
|
||||
Output ""
|
||||
if ! CheckFTS "Auto "; then
|
||||
Output ""
|
||||
Output "FTS indexes are damaged. Attempting automatic FTS rebuild..."
|
||||
WriteLog "Auto - FTS damaged, attempting rebuild"
|
||||
|
||||
if DoFTSRebuild; then
|
||||
WriteLog "Auto - FTS Rebuild - PASS"
|
||||
Output "FTS rebuild successful."
|
||||
else
|
||||
WriteLog "Auto - FTS Rebuild - FAIL"
|
||||
Output "FTS rebuild failed. You may need to run 'reindex' command manually."
|
||||
# Don't fail auto entirely - main DB repair succeeded
|
||||
fi
|
||||
else
|
||||
WriteLog "Auto - FTS Check - PASS"
|
||||
fi
|
||||
|
||||
# All good to here
|
||||
WriteLog "Auto - COMPLETED"
|
||||
Output "Automatic Check, Repair/optimize, & Index successful."
|
||||
Output "Automatic Check, Repair/optimize, Index, & FTS check successful."
|
||||
;;
|
||||
|
||||
|
||||
@@ -2350,6 +2603,14 @@ do
|
||||
WriteLog "Check - FAIL"
|
||||
CheckedDB=0
|
||||
fi
|
||||
|
||||
# Also check FTS indexes (these can be damaged even when integrity_check passes)
|
||||
Output ""
|
||||
if ! CheckFTS "Check "; then
|
||||
Output ""
|
||||
Output "NOTE: FTS indexes are damaged but main database structure is OK."
|
||||
Output " Use 'reindex' (option 6) or 'automatic' (option 2) to rebuild."
|
||||
fi
|
||||
;;
|
||||
|
||||
# Vacuum
|
||||
@@ -2413,6 +2674,24 @@ do
|
||||
# Now index
|
||||
if DoIndex ; then
|
||||
WriteLog "Reindex - PASS"
|
||||
|
||||
# Check FTS indexes and repair if damaged
|
||||
Output ""
|
||||
if ! CheckFTS "Reindex"; then
|
||||
Output ""
|
||||
Output "FTS indexes are damaged. Rebuilding..."
|
||||
WriteLog "Reindex - FTS damaged, attempting rebuild"
|
||||
|
||||
if DoFTSRebuild; then
|
||||
WriteLog "Reindex - FTS Rebuild - PASS"
|
||||
Output "FTS rebuild successful."
|
||||
else
|
||||
WriteLog "Reindex - FTS Rebuild - FAIL"
|
||||
Output "FTS rebuild failed."
|
||||
fi
|
||||
else
|
||||
WriteLog "Reindex - FTS Check - PASS"
|
||||
fi
|
||||
else
|
||||
WriteLog "Reindex - FAIL"
|
||||
fi
|
||||
@@ -2494,6 +2773,9 @@ do
|
||||
[ $CheckedDB -eq 0 ] && Output " Databases are not checked, Status unknown."
|
||||
[ $CheckedDB -eq 1 ] && [ $Damaged -eq 0 ] && Output " Databases are OK."
|
||||
[ $CheckedDB -eq 1 ] && [ $Damaged -eq 1 ] && Output " Databases were checked and are damaged."
|
||||
[ $CheckedFTS -eq 0 ] && Output " FTS indexes are not checked, Status unknown."
|
||||
[ $CheckedFTS -eq 1 ] && [ $FTSDamaged -eq 0 ] && Output " FTS indexes are OK."
|
||||
[ $CheckedFTS -eq 1 ] && [ $FTSDamaged -eq 1 ] && Output " FTS indexes are damaged."
|
||||
Output ""
|
||||
;;
|
||||
|
||||
|
||||
33
README.md
33
README.md
@@ -29,15 +29,15 @@ If sufficient privleges exist (root), and supported by the environment, the opti
|
||||
|
||||
The following commands (or their number), listed in alphabetical order, are accepted as input.
|
||||
```
|
||||
AUTO(matic) - Automatically check, repair/optimize, and reindex the databases in one step.
|
||||
CHEC(k) - Check the main and blob databases integrity
|
||||
AUTO(matic) - Automatically check, repair/optimize, reindex, and FTS rebuild in one step.
|
||||
CHEC(k) - Check the main and blob databases integrity (includes FTS index check)
|
||||
DEFL(ate) - Deflate a bloated PMS database (faulty statistics data)
|
||||
EXIT - Exit the utility
|
||||
IGNOre/HONOr - Ignore/Honor constraint errors when IMPORTing additional data into DB.
|
||||
IMPO(rt) - Import viewstate / watch history from another database
|
||||
PRUN(e) - Prune (remove) old image files from transcoder cache diretory
|
||||
PURG(e) - Purge (delete) all temporary files left behind by PMS & the transcoder from the temp directory
|
||||
REIN(dex) - Rebuild the database indexes
|
||||
REIN(dex) - Rebuild the database indexes (includes FTS indexes)
|
||||
REPL(ace) - Replace the existing databases with a PMS-generated backup
|
||||
SHOW - Show the log file
|
||||
STAR(t) - Start PMS (not available on all platforms)
|
||||
@@ -58,8 +58,8 @@ If sufficient privleges exist (root), and supported by the environment, the opti
|
||||
Select
|
||||
|
||||
1 - 'stop' - Stop PMS.
|
||||
2 - 'automatic' - Check, Repair/Optimize, and Reindex Database in one step.
|
||||
3 - 'check' - Perform integrity check of database.
|
||||
2 - 'automatic' - Check, Repair/Optimize, Reindex, and FTS rebuild in one step.
|
||||
3 - 'check' - Perform integrity check of database and FTS indexes.
|
||||
4 - 'vacuum' - Remove empty space from database without optimizing.
|
||||
5 - 'repair' - Repair/Optimize databases.
|
||||
6 - 'reindex' - Rebuild database indexes.
|
||||
@@ -265,6 +265,17 @@ These examples
|
||||
than you were before. In this case, UNDO then Repair. Undo can only undo the single most-recent action.
|
||||
(Note: In a future release, you will be able to 'undo' every action taken until the DBs are in their original state)
|
||||
|
||||
G. HTTP 500 errors when adding to collections / FTS index corruption
|
||||
This occurs when standard integrity_check passes but FTS (Full-Text Search) indexes are corrupted.
|
||||
Symptoms: Adding items to collections fails, updating metadata fails, "database disk image is malformed"
|
||||
during UPDATE operations even though Check reports databases are OK.
|
||||
|
||||
1. (3) Check - Will show "FTS index damaged" message
|
||||
2. (6) Reindex - Rebuild indexes including FTS
|
||||
3. (99) Exit
|
||||
|
||||
Alternatively, use (2) Automatic which will detect and repair FTS issues automatically.
|
||||
|
||||
Special considerations:
|
||||
|
||||
1. As stated above, this utility requires PMS to be stopped in order to do what it does.
|
||||
@@ -667,6 +678,13 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases#
|
||||
|
||||
Checks the integrity of the Plex main and blobs databases.
|
||||
|
||||
Also performs FTS (Full-Text Search) index integrity checks. FTS indexes can become
|
||||
corrupted even when standard integrity checks pass, causing operations like adding
|
||||
items to collections to fail with "database disk image is malformed" errors.
|
||||
|
||||
If FTS corruption is detected, use 'reindex' (option 6) or 'automatic' (option 2)
|
||||
to rebuild the FTS indexes.
|
||||
|
||||
### Deflate
|
||||
|
||||
Repairs a known error in the PMS main database "statistics_bandwidth" table.
|
||||
@@ -710,6 +728,11 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases#
|
||||
Rebuilds the database indexes after an import, repair, or replace operation.
|
||||
These indexes are used by PMS for searching (both internally and your typed searches)
|
||||
|
||||
Also checks FTS (Full-Text Search) index integrity and rebuilds if corruption is detected.
|
||||
FTS indexes can become corrupted even when standard integrity checks pass, causing
|
||||
"database disk image is malformed" errors during UPDATE operations (e.g., adding items
|
||||
to collections, updating metadata).
|
||||
|
||||
### Repair
|
||||
|
||||
Extracts/recovers all the usable data from the existing databases into text (SQL ascii) form.
|
||||
|
||||
Reference in New Issue
Block a user