Merge pull request #123 from ChuckPa/chuckpa/add-cache-cleaning

Add 'purge' & Environment variable support.

DBRepair.sh

@@ -2,12 +2,12 @@
 #########################################################################
 # Plex Media Server database check and repair utility script.           #
 # Maintainer: ChuckPa                                                   #
-# Version:    v1.02.01                                                  #
+# Version:    v1.03.00                                                  #
-# Date:       12-Jan-2024                                               #
+# Date:       17-Jan-2024                                               #
 #########################################################################
 
 # Version for display purposes
-Version="v1.02.01"
+Version="v1.03.00"
 
 # Flag when temp files are to be retained
 Retain=0
@@ -26,8 +26,8 @@ RootRequired=1
 # By default, Errors are fatal.
 IgnoreErrors=0
 
-# By default, Duplicate view states not purged
+# By default, Duplicate view states not Removed
-PurgeDuplicates=0
+RemoveDuplicates=0
 
 # Keep track of how many times the user's hit enter with no command (implied EOF)
 NullCommands=0
@@ -41,6 +41,7 @@ TimeStamp="$(date "+%Y-%m-%d_%H.%M.%S")"
 # Initialize global runtime variables
 CheckedDB=0
 Damaged=0
+DbPageSize=0
 Fail=0
 HaveStartStop=0
 HostType=""
@@ -805,6 +806,38 @@ DoUndo(){
 
 }
 
+##### DoSetPageSize
+DoSetPageSize() {
+
+  # If DBREPAIR_PAGESIZE variable exists, validate it.
+  [ "$DBREPAIR_PAGESIZE" = "" ] && return
+
+  # Is it a valid positive integer ?
+  if [ "$DBREPAIR_PAGESIZE" != "$(echo "$DBREPAIR_PAGESIZE" | sed 's/[^0-9]*//g')" ]; then
+    WriteLog "SetPageSize - ERROR: DBREPAIR_PAGESIZE is not a valid integer. Ignoring '$DBREPAIR_PAGESIZE'"
+    Output "SetPageSize - ERROR: DBREPAIR_PAGESIZE is not a valid integer. Ignoring '$DBREPAIR_PAGESIZE'"
+    return
+  fi
+
+  # Make certain it's a multiple of 1024 and gt 0
+  DbPageSize=$(expr $DBREPAIR_PAGESIZE + 1023)
+  DbPageSize=$(expr $DbPageSize / 1024)
+  DbPageSize=$(expr $DbPageSize \* 1024)
+
+
+  # Must be compliant
+  [ $DbPageSize -le     0 ] && return
+  [ $DbPageSize -gt 65536 ] && DbPageSize=65536 && WriteLog "SetPageSize - DBREPAIR_PAGESIZE too large. Reducing."
+
+  Output "Setting Plex SQLite page size ($DbPageSize)"
+  WriteLog  "SetPageSize - Setting Plex SQLite page_size: $DbPageSize"
+
+
+# Create DB with desired page size
+"$PLEX_SQLITE" "$1" "PRAGMA page_size=${DbPageSize}; VACUUM;"
+
+}
+
 ##### DoRepair
 DoRepair() {
 
@@ -881,6 +914,7 @@ DoRepair() {
 
     # Library and blobs successfully exported, create new
     Output "Importing Main DB."
+    DoSetPageSize "$TMPDIR/$CPPL.db-REPAIR-$TimeStamp"
     "$PLEX_SQLITE" "$TMPDIR/$CPPL.db-REPAIR-$TimeStamp" < "$TMPDIR/library.plexapp.sql-$TimeStamp"
     Result=$?
     [ $IgnoreErrors -eq 1 ] && Result=0
@@ -894,6 +928,7 @@ DoRepair() {
     fi
 
     Output "Importing Blobs DB."
+    DoSetPageSize "$TMPDIR/$CPPL.blobs.db-REPAIR-$TimeStamp"
     "$PLEX_SQLITE" "$TMPDIR/$CPPL.blobs.db-REPAIR-$TimeStamp" < "$TMPDIR/blobs.plexapp.sql-$TimeStamp"
     Result=$?
     [ $IgnoreErrors -eq 1 ] && Result=0
@@ -1031,8 +1066,8 @@ DoReplace() {
 
       # Make certain there is ample free space
       if ! FreeSpaceAvailable ; then
-        Output "ERROR:  Insufficient free space available on $AppSuppDir.  Cannot continue
+        Output "ERROR:  Insufficient free space available on $AppSuppDir.  Cannot continue"
-        WriteLog "REPLACE -  Insufficient free space available on $AppSuppDir.  Aborted.
+        WriteLog "REPLACE -  Insufficient free space available on $AppSuppDir.  Aborted."
         return 1
       fi
 
@@ -1303,10 +1338,11 @@ DoImport(){
 
   # Import viewstates into working copy (Ignore constraint errors during import)
   printf 'Importing Viewstate & History data...'
+  DoSetPageSize "$TMPDIR/$CPPL.db-IMPORT-$TimeStamp"
   "$PLEX_SQLITE" "$TMPDIR/$CPPL.db-IMPORT-$TimeStamp" < "$TMPDIR/Viewstate.sql-$TimeStamp" 2> /dev/null
 
-  # Purge duplicates (violations of unique constraint)
+  # Remove duplicates (violations of unique constraint)
-  if [ $PurgeDuplicates -eq 1 ]; then
+  if [ $RemoveDuplicates -eq 1 ]; then
    cat <<EOF | "$PLEX_SQLITE" "$TMPDIR/$CPPL.db-IMPORT-$TimeStamp"
     DELETE FROM metadata_item_settings
     WHERE rowid NOT IN
@@ -1437,7 +1473,7 @@ DoOptions() {
     Opt="$(echo $i | cut -c1-2 | tr [A-Z] [a-z])"
     [ "$Opt" = "-i" ] && IgnoreErrors=1 && WriteLog "Opt: Database error checking ignored."
     [ "$Opt" = "-f" ] && IgnoreErrors=1 && WriteLog "Opt: Database error checking ignored."
-    [ "$Opt" = "-p" ] && PurgeDuplicates=1 && WriteLog "Opt: Purge duplidate watch history viewstates."
+    [ "$Opt" = "-p" ] && RemoveDuplicates=1 && WriteLog "Opt: Remove duplidate watch history viewstates."
   done
 }
 
@@ -1487,6 +1523,53 @@ DownloadAndUpdate() {
     return 1
 }
 
+# Prune old jpg files from the PhotoTranscoder directory (> 30 days -or- DBREPAIR_CACHEAGE days)
+DoPrunePhotoTranscoder() {
+
+  TransCacheDir="$AppSuppDir/Plex Media Server/Cache/PhotoTranscoder"
+  PruneIt=0
+
+  # Use default cache age of 30 days
+  CacheAge=30
+
+  # Does DBREPAIR_CACHEAGE exist and is it a valid positive integer ?
+  if [ "$DBREPAIR_CACHEAGE" != "" ]; then
+    if [ "$DBREPAIR_CACHEAGE" != "$(echo "$DBREPAIR_CACHEAGE" | sed 's/[^0-9]*//g')" ]; then
+      WriteLog "PrunePhotoTranscoder  - ERROR: DBREPAIR_CACHEAGE is not a valid integer. Ignoring '$DBREPAIR_CACHEAGE'"
+      Output "PrunePhotoTranscoder  - ERROR: DBREPAIR_CACHEAGE is not a valid integer. Ignoring '$DBREPAIR_CACHEAGE'"
+      return
+    else
+      CacheAge=$DBREPAIR_CACHEAGE
+    fi
+  fi
+
+  # If scripted / command line options, clean automatically
+  if [ $Scripted -eq 1 ]; then
+    PruneIt=1
+  else
+    Output "Counting how many files are more than $CacheAge days old."
+    FileCount=$(find "$TransCacheDir" \( -name \*.jpg -o -name \*.jpeg -o -name \*.png \) -mtime +${CacheAge} -print | wc -l)
+
+    # If nothing found, continue back to the menu
+    [ $FileCount -eq 0 ] && Output "No files found to prune." && return
+
+    # Ask if we should remove it
+    if ConfirmYesNo "OK to prune $FileCount files? "; then
+        PruneIt=1
+    fi
+  fi
+
+  # Prune old the jpgs/jpegs ?
+  if [ $PruneIt -eq 1 ]; then
+    Output "Pruning started."
+    WriteLog "Prune   - Removing $FileCount files over $CacheAge days old."
+    find "$TransCacheDir" \( -name \*.jpg -o -name \*.jpeg -o -name \*.png \) -mtime +${CacheAge} -exec rm -f {} \;
+    Output "Pruning completed."
+    WriteLog "Prune   - PASS."
+  fi
+
+}
+
 #############################################################
 #         Main utility begins here                          #
 #############################################################
@@ -1627,6 +1710,7 @@ do
       echo " 12 - 'undo'      - Undo last successful command."
 
       echo ""
+      echo " 21 - 'prune'     - Prune (remove) old image files (jpeg,jpg,png) from PhotoTranscoder cache."
       [ $IgnoreErrors -eq 0 ] && echo " 42 - 'ignore'    - Ignore duplicate/constraint errors."
       [ $IgnoreErrors -eq 1 ] && echo " 42 - 'honor'     - Honor all database errors."
 
@@ -1933,6 +2017,23 @@ do
          DoUndo
          ;;
 
+
+      # Remove (prune) unused image files from PhotoTranscoder Cache > 30 days old.
+      21|prun*|remo*)
+
+        # Check if PMS running
+        if IsRunning; then
+          WriteLog "Prune   - FAIL - PMS runnning"
+          Output   "Unable to prune PhotoTranscoder cache.  PMS is running."
+          continue
+        fi
+
+        WriteLog "Prune   - START"
+        DoPrunePhotoTranscoder
+        WriteLog "Prune   - PASS"
+        ;;
+
+
       # Ignore/Honor errors
       42|igno*|hono*)
 

README.md

@@ -619,6 +619,16 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases#
   Imports (raw) watch history from another PMS database without ability to check validity
   ( This can have side effects of "negative watch count" being displayed.   Caution is advised. )
 
+### Prune (Remove)
+
+  Checks the PhotoTransoder cache directory for JPG, JPEG, and PNG files older than 30 days and removes them.
+  Under normal operation,  PMS manages this automatically.
+  Under certain conditions, PMS will fail to prune them  (which is run during Scheduled Maintenance)
+  This command allows you to manually remove what PMS would do normally during that Scheduled Maintenance.
+
+  Use of this command will depend on the usage.
+
+  For now, Both forms "Prune" and "Remove" are accepted.
 
 ### Reindex
 
@@ -682,6 +692,10 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases#
   After DB tasks are completed, and you've exited the container,  restart it normally through
   your normal 'docker start' mechanism.
 
+  If your container (Image) supports start/stop ,  it will be shown in the menu for you to use.
+  If not,  you'll need to disable health checks before safely running this tool.
+
+
 ### Undo
 
   Undo allows you to "Undo" the last Import, Repair, Replace, or Vacuum command.
@@ -693,6 +707,118 @@ root@lizum:/sata/plex/Plex Media Server/Plug-in Support/Databases#
   Instructs SQLite to remove the empty/deleted records and gaps from the databases.
   This is most beneficial after deleting whole library sections.
 
+  For most users, the "automatic" command is the best method. It will regenerate the SQLite indexes
+  as part of the process.
+
+# Environment Variables
+
+  DBRepair now supports the use of environment variables to allow customization of some operations.
+
+####  WARNING:  Use of these variables may adverse impact PMS operation or performance.  USE WITH CAUTION.
+
+## DBREPAIR_CACHEAGE - Specify the maximum age for PhotoTrancoder Cache images to be retained
+
+  Default DBREPAIR_CACHEAGE is set at 30 days.
+
+  You may override this by setting  DBREPAIR_CACHEAGE=N,  where N is the number of days worth of cache image
+  you wish to retain.
+
+  When using interactively,  DBRepair will prompt you to confirm OK to remove and show you the cache age
+
+  Example:  export DBREPAIR_CACHEAGE=20
+  ```
+  Enter command # -or- command name (4 char min) : remove
+
+  Counting how many files are more than 20 days old.
+  OK to prune 4497 files?  (Y/N) ?
+  ```
+
+## DBREPAIR_PAGESIZE - Allows setting the Plex SQLite 'page_size'.
+
+  Normal Linux (ext4, xfs) filesystems do not need this customization because the filesystem block size = 4096.
+  ZFS users sometimes need to customize their datasets to improve I/O performance (HDDs vs SSDs).
+  This capability allows them to compensate for some of those losses.
+
+  If present, sets the Plex SQLite 'page_size' for both Main and Blobs databases.
+  If not present, the default page size for the host OS is used (typically 4096 to match the OS page size).
+
+  When in use, you will see the message:  "Setting Plex SQLite page size ($DbPageSize)"
+  This will be shown on the console output and reported in the logfile.
+
+  If the value is invalid, an error will be printed and recorded in the logfile.
+  If the value is too large, it will be reduced to the SQLite maximum of 65536.
+
+### Constraints:
+
+    1.  Must be a multiple of 1024  (per SQLite 3.12.0 documentation).
+        Any value provided willl be forced to be a multiple of 1024.
+        This may or may not be the value you intended.  Caution is advised.
+
+    2.  May not exceed 65536  (per SQLite 3.12.0 documentation).
+        Any value exceeding 65536 will be truncated to 65536.
+
+### Management
+
+  If you attempt to optimize your database but find the resultant performance is not to your liking,
+  you may try another value and run "automatic" again.
+
+  If you ultimately decide to run with the default values,
+  1.  Remove the environment variable.
+  2.  Run DBRepair again using "automatic".  Your databases will revert to the host OS's default.
+
+### Usage:  (Linux example shown)
+
+```
+# export DBREPAIR_PAGESIZE=65534
+# ./DBRepair.sh stop auto start exit
+
+
+
+      Plex Media Server Database Repair Utility (Ubuntu 22.04.3 LTS)
+                       Version v1.02.99
+
+
+[2024-01-14 17.25.35] Stopping PMS.
+[2024-01-14 17.25.35] Stopped PMS.
+
+[2024-01-14 17.25.35] Automatic Check,Repair,Index started.
+[2024-01-14 17.25.35]
+[2024-01-14 17.25.35] Checking the PMS databases
+[2024-01-14 17.25.48] Check complete.  PMS main database is OK.
+[2024-01-14 17.25.48] Check complete.  PMS blobs database is OK.
+[2024-01-14 17.25.48]
+[2024-01-14 17.25.48] Exporting current databases using timestamp: 2024-01-14_17.25.35
+[2024-01-14 17.25.48] Exporting Main DB
+[2024-01-14 17.25.59] Exporting Blobs DB
+[2024-01-14 17.26.00] Successfully exported the main and blobs databases.  Proceeding to import into new databases.
+[2024-01-14 17.26.00] Importing Main DB.
+[2024-01-14 17.26.00] Setting Plex SQLite page size (65536)
+[2024-01-14 17.26.29] Importing Blobs DB.
+[2024-01-14 17.26.29] Setting Plex SQLite page size (65536)
+[2024-01-14 17.26.30] Successfully imported databases.
+[2024-01-14 17.26.30] Verifying databases integrity after importing.
+[2024-01-14 17.27.43] Verification complete.  PMS main database is OK.
+[2024-01-14 17.27.43] Verification complete.  PMS blobs database is OK.
+[2024-01-14 17.27.43] Saving current databases with '-BACKUP-2024-01-14_17.25.35'
+[2024-01-14 17.27.43] Making repaired databases active
+[2024-01-14 17.27.43] Repair complete. Please check your library settings and contents for completeness.
+[2024-01-14 17.27.43] Recommend:  Scan Files and Refresh all metadata for each library section.
+[2024-01-14 17.27.43]
+[2024-01-14 17.27.43] Backing up of databases
+[2024-01-14 17.27.43] Backup current databases with '-BACKUP-2024-01-14_17.27.43' timestamp.
+[2024-01-14 17.27.44] Reindexing main database
+[2024-01-14 17.28.08] Reindexing main database successful.
+[2024-01-14 17.28.08] Reindexing blobs database
+[2024-01-14 17.28.08] Reindexing blobs database successful.
+[2024-01-14 17.28.08] Reindex complete.
+[2024-01-14 17.28.08] Automatic Check, Repair/optimize, & Index successful.
+
+[2024-01-14 17.28.08] Starting PMS.
+[2024-01-14 17.28.08] Started PMS
+
+#
+```
+
 # Special considerations - Synology DSM 7
 
   Using DBRepair on Synology DSM 7 systems with Task Scheduler requires special handling.

ReleaseNotes

@@ -8,6 +8,27 @@
 ![Maintenance](https://img.shields.io/badge/Maintained-Yes-green.svg)
 
 # Release Info:
+v1.03.00
+
+  1. New command: Purge -   Removes old image files from the Trancoder cache.
+                            These are normally removed during scheduled maintenance.
+                            If scheduled maintenance doesn't complete all tasks,  they won't get removed
+                            and will build up.
+
+                            This command allows you to remove them.
+
+                            The default age is 30 days.  Files older than 30 days will be purged.
+
+  2. Introduces Environment variable support (customizing) DBRepair operation.
+
+    - DBREPAIR_PAGESIZE:    DBREPAIR_PAGESIZE=N  (where N is a multiple of 1024, less equal 65536)
+                            This sets the Plex SQLite database page_size during optimization operations.
+                            It's most beneficial on ZFS filesystems to allow tuning to the Dataset size.
+
+    - DBREPAIR_CACHEAGE:    DBREPAIR_CACHEAGE=N  (where N is the max age, in Days) to retain when purging
+                            older image files in the Cache/PhotoTranscoder directory.
+                            This variable allows you to set the purge age (greater than "N" days).
+
 v1.02.01
 
   - Minor fix to logging.  One line of output at the console was not present in the logfile