From 2a8a8e242bd49aed5cbf049425b280147d83eed3 Mon Sep 17 00:00:00 2001 From: ChuckPa Date: Wed, 17 Jan 2024 00:30:47 -0500 Subject: [PATCH] Add 'purge' & Environment variable support. PlexDBRepair - release v01.03.00 1. Purge command ('piurge') - Remove old image files from Transcode cache 2. Environment variables support to tailor operation - DBREPAIR_CACHEAGE - Set max age of files to retain - DBREPAIR_PAGESIZE - Set Plex SQLite database page_size --- DBRepair.sh | 121 +++++++++++++++++++++++++++++++++++++++++++++---- README.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++ ReleaseNotes | 21 +++++++++ 3 files changed, 258 insertions(+), 10 deletions(-) diff --git a/DBRepair.sh b/DBRepair.sh index 6c9d0d3..0a00573 100755 --- a/DBRepair.sh +++ b/DBRepair.sh @@ -2,12 +2,12 @@ ######################################################################### # Plex Media Server database check and repair utility script. # # Maintainer: ChuckPa # -# Version: v1.02.01 # -# Date: 12-Jan-2024 # +# Version: v1.03.00 # +# Date: 17-Jan-2024 # ######################################################################### # Version for display purposes -Version="v1.02.01" +Version="v1.03.00" # Flag when temp files are to be retained Retain=0 @@ -26,8 +26,8 @@ RootRequired=1 # By default, Errors are fatal. IgnoreErrors=0 -# By default, Duplicate view states not purged -PurgeDuplicates=0 +# By default, Duplicate view states not Removed +RemoveDuplicates=0 # Keep track of how many times the user's hit enter with no command (implied EOF) NullCommands=0 @@ -41,6 +41,7 @@ TimeStamp="$(date "+%Y-%m-%d_%H.%M.%S")" # Initialize global runtime variables CheckedDB=0 Damaged=0 +DbPageSize=0 Fail=0 HaveStartStop=0 HostType="" @@ -805,6 +806,38 @@ DoUndo(){ } +##### DoSetPageSize +DoSetPageSize() { + + # If DBREPAIR_PAGESIZE variable exists, validate it. + [ "$DBREPAIR_PAGESIZE" = "" ] && return + + # Is it a valid positive integer ? + if [ "$DBREPAIR_PAGESIZE" != "$(echo "$DBREPAIR_PAGESIZE" | sed 's/[^0-9]*//g')" ]; then + WriteLog "SetPageSize - ERROR: DBREPAIR_PAGESIZE is not a valid integer. Ignoring '$DBREPAIR_PAGESIZE'" + Output "SetPageSize - ERROR: DBREPAIR_PAGESIZE is not a valid integer. Ignoring '$DBREPAIR_PAGESIZE'" + return + fi + + # Make certain it's a multiple of 1024 and gt 0 + DbPageSize=$(expr $DBREPAIR_PAGESIZE + 1023) + DbPageSize=$(expr $DbPageSize / 1024) + DbPageSize=$(expr $DbPageSize \* 1024) + + + # Must be compliant + [ $DbPageSize -le 0 ] && return + [ $DbPageSize -gt 65536 ] && DbPageSize=65536 && WriteLog "SetPageSize - DBREPAIR_PAGESIZE too large. Reducing." + + Output "Setting Plex SQLite page size ($DbPageSize)" + WriteLog "SetPageSize - Setting Plex SQLite page_size: $DbPageSize" + + +# Create DB with desired page size +"$PLEX_SQLITE" "$1" "PRAGMA page_size=${DbPageSize}; VACUUM;" + +} + ##### DoRepair DoRepair() { @@ -881,6 +914,7 @@ DoRepair() { # Library and blobs successfully exported, create new Output "Importing Main DB." + DoSetPageSize "$TMPDIR/$CPPL.db-REPAIR-$TimeStamp" "$PLEX_SQLITE" "$TMPDIR/$CPPL.db-REPAIR-$TimeStamp" < "$TMPDIR/library.plexapp.sql-$TimeStamp" Result=$? [ $IgnoreErrors -eq 1 ] && Result=0 @@ -894,6 +928,7 @@ DoRepair() { fi Output "Importing Blobs DB." + DoSetPageSize "$TMPDIR/$CPPL.blobs.db-REPAIR-$TimeStamp" "$PLEX_SQLITE" "$TMPDIR/$CPPL.blobs.db-REPAIR-$TimeStamp" < "$TMPDIR/blobs.plexapp.sql-$TimeStamp" Result=$? [ $IgnoreErrors -eq 1 ] && Result=0 @@ -1031,8 +1066,8 @@ DoReplace() { # Make certain there is ample free space if ! FreeSpaceAvailable ; then - Output "ERROR: Insufficient free space available on $AppSuppDir. Cannot continue - WriteLog "REPLACE - Insufficient free space available on $AppSuppDir. Aborted. + Output "ERROR: Insufficient free space available on $AppSuppDir. Cannot continue" + WriteLog "REPLACE - Insufficient free space available on $AppSuppDir. Aborted." return 1 fi @@ -1303,10 +1338,11 @@ DoImport(){ # Import viewstates into working copy (Ignore constraint errors during import) printf 'Importing Viewstate & History data...' + DoSetPageSize "$TMPDIR/$CPPL.db-IMPORT-$TimeStamp" "$PLEX_SQLITE" "$TMPDIR/$CPPL.db-IMPORT-$TimeStamp" < "$TMPDIR/Viewstate.sql-$TimeStamp" 2> /dev/null - # Purge duplicates (violations of unique constraint) - if [ $PurgeDuplicates -eq 1 ]; then + # Remove duplicates (violations of unique constraint) + if [ $RemoveDuplicates -eq 1 ]; then cat < 30 days -or- DBREPAIR_CACHEAGE days) +DoPrunePhotoTranscoder() { + + TransCacheDir="$AppSuppDir/Plex Media Server/Cache/PhotoTranscoder" + PruneIt=0 + + # Use default cache age of 30 days + CacheAge=30 + + # Does DBREPAIR_CACHEAGE exist and is it a valid positive integer ? + if [ "$DBREPAIR_CACHEAGE" != "" ]; then + if [ "$DBREPAIR_CACHEAGE" != "$(echo "$DBREPAIR_CACHEAGE" | sed 's/[^0-9]*//g')" ]; then + WriteLog "PrunePhotoTranscoder - ERROR: DBREPAIR_CACHEAGE is not a valid integer. Ignoring '$DBREPAIR_CACHEAGE'" + Output "PrunePhotoTranscoder - ERROR: DBREPAIR_CACHEAGE is not a valid integer. Ignoring '$DBREPAIR_CACHEAGE'" + return + else + CacheAge=$DBREPAIR_CACHEAGE + fi + fi + + # If scripted / command line options, clean automatically + if [ $Scripted -eq 1 ]; then + PruneIt=1 + else + Output "Counting how many files are more than $CacheAge days old." + FileCount=$(find "$TransCacheDir" \( -name \*.jpg -o -name \*.jpeg -o -name \*.png \) -mtime +${CacheAge} -print | wc -l) + + # If nothing found, continue back to the menu + [ $FileCount -eq 0 ] && Output "No files found to prune." && return + + # Ask if we should remove it + if ConfirmYesNo "OK to prune $FileCount files? "; then + PruneIt=1 + fi + fi + + # Prune old the jpgs/jpegs ? + if [ $PruneIt -eq 1 ]; then + Output "Pruning started." + WriteLog "Prune - Removing $FileCount files over $CacheAge days old." + find "$TransCacheDir" \( -name \*.jpg -o -name \*.jpeg -o -name \*.png \) -mtime +${CacheAge} -exec rm -f {} \; + Output "Pruning completed." + WriteLog "Prune - PASS." + fi + +} + ############################################################# # Main utility begins here # ############################################################# @@ -1627,6 +1710,7 @@ do echo " 12 - 'undo' - Undo last successful command." echo "" + echo " 21 - 'prune' - Prune (remove) old image files (jpeg,jpg,png) from PhotoTranscoder cache." [ $IgnoreErrors -eq 0 ] && echo " 42 - 'ignore' - Ignore duplicate/constraint errors." [ $IgnoreErrors -eq 1 ] && echo " 42 - 'honor' - Honor all database errors." @@ -1933,6 +2017,23 @@ do DoUndo ;; + + # Remove (prune) unused image files from PhotoTranscoder Cache > 30 days old. + 21|prun*|remo*) + + # Check if PMS running + if IsRunning; then + WriteLog "Prune - FAIL - PMS runnning" + Output "Unable to prune PhotoTranscoder cache. PMS is running." + continue + fi + + WriteLog "Prune - START" + DoPrunePhotoTranscoder + WriteLog "Prune - PASS" + ;; + + # Ignore/Honor errors 42|igno*|hono*) diff --git a/README.md b/README.md index 3ba4711..107a74a 100644 --- a/README.md +++ b/README.md @@ -619,6 +619,16 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases# Imports (raw) watch history from another PMS database without ability to check validity ( This can have side effects of "negative watch count" being displayed. Caution is advised. ) +### Prune (Remove) + + Checks the PhotoTransoder cache directory for JPG, JPEG, and PNG files older than 30 days and removes them. + Under normal operation, PMS manages this automatically. + Under certain conditions, PMS will fail to prune them (which is run during Scheduled Maintenance) + This command allows you to manually remove what PMS would do normally during that Scheduled Maintenance. + + Use of this command will depend on the usage. + + For now, Both forms "Prune" and "Remove" are accepted. ### Reindex @@ -682,6 +692,10 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases# After DB tasks are completed, and you've exited the container, restart it normally through your normal 'docker start' mechanism. + If your container (Image) supports start/stop , it will be shown in the menu for you to use. + If not, you'll need to disable health checks before safely running this tool. + + ### Undo Undo allows you to "Undo" the last Import, Repair, Replace, or Vacuum command. @@ -693,6 +707,118 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases# Instructs SQLite to remove the empty/deleted records and gaps from the databases. This is most beneficial after deleting whole library sections. + For most users, the "automatic" command is the best method. It will regenerate the SQLite indexes + as part of the process. + +# Environment Variables + + DBRepair now supports the use of environment variables to allow customization of some operations. + +#### WARNING: Use of these variables may adverse impact PMS operation or performance. USE WITH CAUTION. + +## DBREPAIR_CACHEAGE - Specify the maximum age for PhotoTrancoder Cache images to be retained + + Default DBREPAIR_CACHEAGE is set at 30 days. + + You may override this by setting DBREPAIR_CACHEAGE=N, where N is the number of days worth of cache image + you wish to retain. + + When using interactively, DBRepair will prompt you to confirm OK to remove and show you the cache age + + Example: export DBREPAIR_CACHEAGE=20 + ``` + Enter command # -or- command name (4 char min) : remove + + Counting how many files are more than 20 days old. + OK to prune 4497 files? (Y/N) ? + ``` + +## DBREPAIR_PAGESIZE - Allows setting the Plex SQLite 'page_size'. + + Normal Linux (ext4, xfs) filesystems do not need this customization because the filesystem block size = 4096. + ZFS users sometimes need to customize their datasets to improve I/O performance (HDDs vs SSDs). + This capability allows them to compensate for some of those losses. + + If present, sets the Plex SQLite 'page_size' for both Main and Blobs databases. + If not present, the default page size for the host OS is used (typically 4096 to match the OS page size). + + When in use, you will see the message: "Setting Plex SQLite page size ($DbPageSize)" + This will be shown on the console output and reported in the logfile. + + If the value is invalid, an error will be printed and recorded in the logfile. + If the value is too large, it will be reduced to the SQLite maximum of 65536. + +### Constraints: + + 1. Must be a multiple of 1024 (per SQLite 3.12.0 documentation). + Any value provided willl be forced to be a multiple of 1024. + This may or may not be the value you intended. Caution is advised. + + 2. May not exceed 65536 (per SQLite 3.12.0 documentation). + Any value exceeding 65536 will be truncated to 65536. + +### Management + + If you attempt to optimize your database but find the resultant performance is not to your liking, + you may try another value and run "automatic" again. + + If you ultimately decide to run with the default values, + 1. Remove the environment variable. + 2. Run DBRepair again using "automatic". Your databases will revert to the host OS's default. + +### Usage: (Linux example shown) + +``` +# export DBREPAIR_PAGESIZE=65534 +# ./DBRepair.sh stop auto start exit + + + + Plex Media Server Database Repair Utility (Ubuntu 22.04.3 LTS) + Version v1.02.99 + + +[2024-01-14 17.25.35] Stopping PMS. +[2024-01-14 17.25.35] Stopped PMS. + +[2024-01-14 17.25.35] Automatic Check,Repair,Index started. +[2024-01-14 17.25.35] +[2024-01-14 17.25.35] Checking the PMS databases +[2024-01-14 17.25.48] Check complete. PMS main database is OK. +[2024-01-14 17.25.48] Check complete. PMS blobs database is OK. +[2024-01-14 17.25.48] +[2024-01-14 17.25.48] Exporting current databases using timestamp: 2024-01-14_17.25.35 +[2024-01-14 17.25.48] Exporting Main DB +[2024-01-14 17.25.59] Exporting Blobs DB +[2024-01-14 17.26.00] Successfully exported the main and blobs databases. Proceeding to import into new databases. +[2024-01-14 17.26.00] Importing Main DB. +[2024-01-14 17.26.00] Setting Plex SQLite page size (65536) +[2024-01-14 17.26.29] Importing Blobs DB. +[2024-01-14 17.26.29] Setting Plex SQLite page size (65536) +[2024-01-14 17.26.30] Successfully imported databases. +[2024-01-14 17.26.30] Verifying databases integrity after importing. +[2024-01-14 17.27.43] Verification complete. PMS main database is OK. +[2024-01-14 17.27.43] Verification complete. PMS blobs database is OK. +[2024-01-14 17.27.43] Saving current databases with '-BACKUP-2024-01-14_17.25.35' +[2024-01-14 17.27.43] Making repaired databases active +[2024-01-14 17.27.43] Repair complete. Please check your library settings and contents for completeness. +[2024-01-14 17.27.43] Recommend: Scan Files and Refresh all metadata for each library section. +[2024-01-14 17.27.43] +[2024-01-14 17.27.43] Backing up of databases +[2024-01-14 17.27.43] Backup current databases with '-BACKUP-2024-01-14_17.27.43' timestamp. +[2024-01-14 17.27.44] Reindexing main database +[2024-01-14 17.28.08] Reindexing main database successful. +[2024-01-14 17.28.08] Reindexing blobs database +[2024-01-14 17.28.08] Reindexing blobs database successful. +[2024-01-14 17.28.08] Reindex complete. +[2024-01-14 17.28.08] Automatic Check, Repair/optimize, & Index successful. + +[2024-01-14 17.28.08] Starting PMS. +[2024-01-14 17.28.08] Started PMS + +# +``` + # Special considerations - Synology DSM 7 Using DBRepair on Synology DSM 7 systems with Task Scheduler requires special handling. diff --git a/ReleaseNotes b/ReleaseNotes index ce1fc61..649e479 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -8,6 +8,27 @@ ![Maintenance](https://img.shields.io/badge/Maintained-Yes-green.svg) # Release Info: +v1.03.00 + + 1. New command: Purge - Removes old image files from the Trancoder cache. + These are normally removed during scheduled maintenance. + If scheduled maintenance doesn't complete all tasks, they won't get removed + and will build up. + + This command allows you to remove them. + + The default age is 30 days. Files older than 30 days will be purged. + + 2. Introduces Environment variable support (customizing) DBRepair operation. + + - DBREPAIR_PAGESIZE: DBREPAIR_PAGESIZE=N (where N is a multiple of 1024, less equal 65536) + This sets the Plex SQLite database page_size during optimization operations. + It's most beneficial on ZFS filesystems to allow tuning to the Dataset size. + + - DBREPAIR_CACHEAGE: DBREPAIR_CACHEAGE=N (where N is the max age, in Days) to retain when purging + older image files in the Cache/PhotoTranscoder directory. + This variable allows you to set the purge age (greater than "N" days). + v1.02.01 - Minor fix to logging. One line of output at the console was not present in the logfile