From 5857063848e0fc81c1cdefa6a58dabfddde451ea Mon Sep 17 00:00:00 2001
From: Retro_Guy <retro.guy@rocksolidbbs.com>
Date: Sun, 10 Apr 2022 04:30:34 +0000
Subject: [PATCH] Clean up overboard to avoid waste of resources

---
 Rocksolid_Light/rocksolid/newsportal.php  |  45 ++++
 Rocksolid_Light/rocksolid/overboard.php   | 300 ++++++++--------------
 Rocksolid_Light/rslight/scripts/nocem.php |  17 +-
 3 files changed, 166 insertions(+), 196 deletions(-)

diff --git a/Rocksolid_Light/rocksolid/newsportal.php b/Rocksolid_Light/rocksolid/newsportal.php
index 1c4d003..b5cfd1f 100644
--- a/Rocksolid_Light/rocksolid/newsportal.php
+++ b/Rocksolid_Light/rocksolid/newsportal.php
@@ -1543,6 +1543,29 @@ function np_get_db_article($article, $group, $makearray=1, $dbh=null) {
     }
 }
 
+function get_poster_name($name) {
+        $fromline = address_decode($name,"nirgendwo");
+        if (!isset($fromline[0]["host"])) $fromline[0]["host"]="";
+          $name_from=$fromline[0]["mailbox"]."@".$fromline[0]["host"];
+          $name_username=$fromline[0]["mailbox"];
+          if (!isset($fromline[0]["personal"])) {
+            $poster_name=$fromline[0]["mailbox"];
+          } else {
+            $poster_name=$fromline[0]["personal"];
+          }
+       if(trim($poster_name) == '') {
+         $fromoutput = explode("<", html_entity_decode($name));
+         if(strlen($fromoutput[0]) < 1) {
+           $poster_name = $fromoutput[1];
+         } else {
+           $poster_name = $fromoutput[0];
+         }
+       }
+       $thisposter['name'] = $poster_name;
+       $thisposter['from'] = $name_from;
+       return($thisposter);
+}
+
 function get_config_value($configfile,$request) {
   global $config_dir;
     
@@ -1616,6 +1639,28 @@ function write_access_log() {
   file_put_contents($accessfile, "\n".format_log_date()." ".$currentPageUrl, FILE_APPEND);
 }
 
+function get_db_data_from_msgid($msgid, $group) {
+      global $spooldir;
+      $database = $spooldir.'/'.$group.'-articles.db3';
+      if(!is_file($database)) {
+        return false;
+      }
+      $articles_dbh = rslight_db_open($database);
+      $articles_query = $articles_dbh->prepare('SELECT * FROM articles WHERE msgid=:messageid');
+      $articles_query->execute(['messageid' => $msgid]);
+      $found = 0;
+      while ($row = $articles_query->fetch()) {
+        $found = 1;
+        break;
+      }
+      $dbh = null;
+      if($found) {
+        return $row;
+      } else {
+        return false;
+      }
+}
+
 function get_data_from_msgid($msgid) {
       global $spooldir;
       $database = $spooldir.'/articles-overview.db3';
diff --git a/Rocksolid_Light/rocksolid/overboard.php b/Rocksolid_Light/rocksolid/overboard.php
index 99e12be..876c263 100755
--- a/Rocksolid_Light/rocksolid/overboard.php
+++ b/Rocksolid_Light/rocksolid/overboard.php
@@ -34,7 +34,6 @@
       $userfile=$spooldir.'/'.strtolower($_COOKIE['mail_name']).'-articleviews.dat';
     }
   }
-  
 if(isset($frames_on) && $frames_on === true) {
 ?>
 <script>
@@ -56,6 +55,7 @@ if (isset($_GET['thisgroup'])) {
 }
 include "head.inc";
 $CONFIG = include($config_file);
+$logfile=$logdir.'/overboard.log';
 
 # How many days old should articles be displayed?
 if (isset($_GET['thisgroup'])) {
@@ -64,11 +64,13 @@ if (isset($_GET['thisgroup'])) {
   $article_age = 30;
 }
 
+$version = 1.1;
+
 # How long in seconds to cache results
 $cachetime = 60;
 
 # Maximum number of articles to show
-$maxdisplay = 500;
+$maxdisplay = 1000;
 
 # How many characters of the body to display per article
 $snippetlength = 240;
@@ -79,6 +81,7 @@ $thissite = '.';
 $groupconfig=$file_groups;
 $cachefile=$spooldir."/".$config_name."-overboard.dat";
 $oldest = (time() - (86400 * $article_age));
+$prune = false;
 
 if (isset($_GET['time'])) {
   $user_time = $_GET['time'];
@@ -86,6 +89,8 @@ if (isset($_GET['time'])) {
     if(($user_time > time()) || ($user_time < $oldest)) {
       unset($user_time);
     }
+  } else {
+    unset($user_time);
   }
 }
 
@@ -104,35 +109,28 @@ if (isset($_GET['thisgroup'])) {
 show_overboard_header($grouplist);
 
 $results=0;
-/* If cache is less than ? seconds old, use it */
+
 if(is_file($cachefile)) {
   $stats = stat($cachefile);
-  $oldest = $stats[9];
-  $cached_overboard = unserialize(file_get_contents($cachefile));
-  if(!isset($user_time)) {
-    if($stats[9] > (time() - $cachetime)) {
-      echo '<table cellspacing="0" width="100%" class="np_results_table">';
-      foreach($cached_overboard as $result) {
-        if(($results % 2) != 0){
-          echo '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
-        } else {
-          echo '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
-        }
-        echo $result;
-        $results++;
-      }
-      show_overboard_footer($stats, $results, true);
-      exit(0);
-    }
-  }
+  $this_overboard = unserialize(file_get_contents($cachefile));
+  $cachedate = ($this_overboard['lastmessage'] - 86400);
+  $oldest = $cachedate;
+} else {
+  $cachedate = ($oldest - 86400);
 }
-//ob_start();
+if($this_overboard['version'] !== $version) {
+  unset($this_overboard);
+  unlink($cachefile);
+  $this_overboard['version'] = $version;
+  $cachedate = ($oldest - 86400);
+}
+
 # Iterate through groups
 
 $database = $spooldir.'/articles-overview.db3';
 $table = 'overview';
 $dbh = rslight_db_open($database, $table);
-$query = $dbh->prepare('SELECT * FROM '.$table.' WHERE newsgroup=:findgroup AND date >= '.$oldest.' ORDER BY date DESC LIMIT '.$maxdisplay);
+$query = $dbh->prepare('SELECT * FROM '.$table.' WHERE newsgroup=:findgroup AND date >= '.$cachedate.' ORDER BY date DESC LIMIT '.$maxdisplay);
 $articles = array();
 $db_articles = array();
 foreach($grouplist as $findgroup) {
@@ -176,12 +174,8 @@ if($CONFIG['article_database'] == '1') {
   }
 }
 krsort($files);
-echo '<table cellspacing="0" width="100%" class="np_results_table">';
 
 foreach($files as $article) {
-    if(!isset($cachedate)) {
-      $cachedate = time();
-    }
     if($CONFIG['article_database'] == '1') {
       $data = explode(':', $article);
       $articledata = np_get_db_article($data[1], $data[0], 0);
@@ -189,16 +183,8 @@ foreach($files as $article) {
       $articledata = file_get_contents($article);
     }
     $bodystart = strpos($articledata, $localeol);
-
     $header = substr($articledata, 0, $bodystart);
-    $body = substr($articledata, $bodystart+1);
-    $body = substr($body, strpos($body, PHP_EOL));
 
-	if(($multi = strpos($body, 'Content-Type: text/plain')) != false) {
-		$bodystart = strpos($body, $localeol);
-		$body = substr($body, $bodystart+1);
-	    $body = substr($body, strpos($body, PHP_EOL));
-	}
     # Find group name and article number
     if($CONFIG['article_database'] == '1') {
       $group = $data[0];
@@ -211,35 +197,14 @@ foreach($files as $article) {
       $groupname = substr($group, 0, $findme);
       $articlenumber = substr($group, $findme+1);
     }
-    # Generate link
-    $url = $thissite."/article-flat.php?id=".$articlenumber."&group="._rawurlencode($groupname)."#".$articlenumber;
-    $groupurl = $thissite."/thread.php?group="._rawurlencode($groupname);
-    preg_match('/Subject:.*/', $header, $subject);
-    $output = explode("Subject: ",$subject[0], 2);
-
-    preg_match('/Date:.*/', $header, $articledate);
-    $dateoutput = explode("Date: ",$articledate[0]);
-    $thisdate = strtotime($dateoutput[1]);
-
-    if(($thisdate > time()) || ($thisdate < $oldest)) {
-      continue;
-    }
-    $local_poster=false;
-    if(preg_match('/X-Rslight-Site:.*/', $header, $site)) {
-      $site_match = explode("X-Rslight-Site: ", $site[0]);
-      preg_match('/Message-ID:.*/', $header, $mid);
-      $mid_match = explode("Message-ID: ",$mid[0]);
-      $rslight_site = $site_match[1];
-      $rslight_mid = $mid_match[1];
-      if(password_verify($CONFIG['thissitekey'].$rslight_mid, $rslight_site)) {
-        $local_poster=true;
-      }
-  }
 
     preg_match('/Message-ID:.*/i', $header, $articleid);
     $getid = explode(": ", $articleid[0]);
-    $thismsgid = hash('crc32', serialize($getid[1]));
-
+    $thismsgid = $getid[1];
+    if(isset($this_overboard['msgids'][$thismsgid])) {
+      continue;
+    }
+    
     preg_match('/References:.*/i', $header, $ref);
     $getrefs = explode(': ', $ref[0]);
     $ref = preg_split("/[\s]+/", $getrefs[1]);
@@ -255,141 +220,102 @@ foreach($files as $article) {
     } else {
       $threadref = false;
     }
-    preg_match('/Content-Transfer-Encoding:.*/', $header, $te);
-    $content_transfer_encoding = explode("Content-Transfer-Encoding: ", $te[0]); 
 
-    preg_match('/.*charset=.*/', $header, $te);
-    $content_type = explode("Content-Type: text/plain; charset=", $te[0]);
-    $date_interval = date("D, j M Y H:i T", strtotime($dateoutput[1]));
-    preg_match('/Content-Transfer-Encoding:.*/', $header, $encoding);
-    $this_encoding = explode("Content-Transfer-Encoding: ", $encoding[0]);
-    if(trim($this_encoding[1]) == "base64") {
-      $body=base64_decode($body);
+    $target = get_data_from_msgid($thismsgid);
+    if($target['date'] > time()) {
+      continue;
     }
-    if($CONFIG['article_database'] == '1') {
-      $articlefrom[0] = $data[3];
-    } else { 
-      preg_match('/From:.*/', htmlspecialchars($header), $articlefrom);
-      $isfrom = explode("From: ", $articlefrom[0]);    
-      $articlefrom[0] = $isfrom[1];
+    if($target['date'] > $this_overboard['lastmessage']) {
+      $this_overboard['lastmessage'] = $target['date'];
     }
-      $fromline = address_decode($articlefrom[0],"nirgendwo");
-          if (!isset($fromline[0]["host"])) $fromline[0]["host"]="";
-          $name_from=$fromline[0]["mailbox"]."@".$fromline[0]["host"];
-          $name_username=$fromline[0]["mailbox"];
-          if (!isset($fromline[0]["personal"])) {
-            $poster_name=$fromline[0]["mailbox"];
-          } else {
-            $poster_name=$fromline[0]["personal"];
-          }
-       if(trim($poster_name) == '') {
-         $fromoutput = explode("<", html_entity_decode($c->name));
-         if(strlen($fromoutput[0]) < 1) {
-           $poster_name = $fromoutput[1];
-         } else {
-           $poster_name = $fromoutput[0];
-         }
-       }
-    $fromoutput[0] = $poster_name;
-    
-    if(($results % 2) != 0){
-	$this_output = '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
-    } else {
-	$this_output = '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
+    if(!isset($this_overboard['threads'][$target['date']])) {
+      $this_overboard['threads'][$target['date']] = $thismsgid;
+      $this_overboard['msgids'][$thismsgid] = $target;
+      if($threadref) {
+        $this_overboard['threadlink'][$thismsgid] = $threadref;
+      }
+      if($results++ > ($maxdisplay - 2)) {
+  	    break;
+      }
     }
-    $this_output = '<p class=np_ob_subject>';
-    if($threadref) {
-      $this_output.= '<b><a href="'.$url.'">'.mb_decode_mimeheader($output[1]).'</a></b><font class="np_ob_group"><a href="article-flat.php?id='.$refid[number].'&group='.rawurlencode($refid[newsgroup]).'#'.$refid[number].'"> (thread)</a></font>'."\r\n"; 
-    } else {
-      $this_output.= '<b><a href="'.$url.'">'.mb_decode_mimeheader($output[1])."</a></b>\r\n";
-    }
-    $this_output.= '</p><p class=np_ob_group>';
-    $this_output.= '<a href="'.$groupurl.'"><span class="visited">'.$groupname.'</span></a>';
-    $this_output.= '</p>';
-    if((isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) && (strpos($fromoutput[0], '@') !== false)) {
-      $poster_name = truncate_email($fromoutput[0]);
-    } else {
-      $poster_name = $fromoutput[0]; 
-    }
-  if($local_poster) {
-    $this_output.= '<p class=np_ob_posted_date>Posted: '.$date_interval.' by: <i>'.create_name_link(mb_decode_mimeheader($poster_name), $name_from).'</i></p>';
-  } else {
-    $this_output.= '<p class=np_ob_posted_date>Posted: '.$date_interval.' by: '.create_name_link(mb_decode_mimeheader($poster_name), $name_from).'</p>'; 
-  }
-    # Try to display useful snippet
-	if($stop=strpos($body, "begin 644 "))
-		$body=substr($body, 0, $stop);
-    $body = quoted_printable_decode($body);
-    $mysnippet = recode_charset($body, $content_type[1], "utf8");
-    if($bodyend=strrpos($mysnippet, "\n---\n")) {
-	$mysnippet = substr($mysnippet, 0, $bodyend);
-    } else {
-	    if($bodyend=strrpos($mysnippet, "\n-- ")) {
-		$mysnippet = substr($mysnippet, 0, $bodyend);
-		} else {
-			if($bodyend=strrpos($mysnippet, "\n.")) {
-				$mysnippet = substr($mysnippet, 0, $bodyend);
-			} 
-		}
-	}
-	$mysnippet = preg_replace('/\n.{0,5}>(.*)/', '', $mysnippet);
-	$snipstart = strpos($mysnippet, ":\n");
-	if(substr_count(trim(substr($mysnippet, 0, $snipstart)), "\n") < 2) {
-		$mysnippet = substr($mysnippet, $snipstart + 1, $snippetlength);
-	} else {
-		$mysnippet = substr($mysnippet, 0, $snippetlength);
-	}
-    $this_output.= "<p class=np_ob_body>".htmlspecialchars($mysnippet, ENT_QUOTES)."</p>\r\n";
-    $this_output.= '</td></tr>';
-    $this_overboard[$thismsgid] = $this_output;
-
-    if($results++ > ($maxdisplay - 2))
-	  break;
 }
-    if(isset($cached_overboard) && isset($this_overboard)) {
-      $new_overboard = array_merge($this_overboard, $cached_overboard);
-      $new_overboard = array_slice($new_overboard, 0, $maxdisplay);
-      if (!isset($user_time)) {
-        file_put_contents($cachefile, serialize($new_overboard));
-      }
-    } elseif(isset($this_overboard)) {
-      $new_overboard = $this_overboard;
-      if (!isset($user_time)) {
-        file_put_contents($cachefile, serialize($new_overboard));
-      }
-    } else {
-      $new_overboard = $cached_overboard;
-    }
-    if(isset($cachedate)) {
-      if (!isset($user_time)) {
-        touch($cachefile, $cachedate);
-      }
-    }
-    $results = 1;
- 
-    foreach($new_overboard as $result) {
-      if(($results % 2) == 0){
-        echo '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
-      } else {
-        echo '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
-      }
-      if (isset($user_time)) {
-        preg_match('/Posted:.*/i', $result, $posted_date);
-        $get_date = explode(': ', $posted_date[0]);
-        $showme = strtotime(trim(substr($get_date[1], 0, -2)));
-        if(($showme > time()) || ($showme < $user_time)) {
-          $results--;
-          break;
+
+file_put_contents($cachefile, serialize($this_overboard));
+if(isset($user_time)) {
+  $oldest = ($user_time - 900);
+} else {
+  $oldest = (time() - (86400 * $article_age));
+}
+$results = display_threads($this_overboard['threads'], $oldest);
+show_overboard_footer(null, $results, null);
+echo '</body></html>';
+expire_overboard($cachefile);
+
+function expire_overboard($cachefile) {
+  global $article_age, $logfile, $config_name, $prune, $this_overboard;
+    if($this_overboard['expire'] < (time() - 86400)) {
+      $prune = true;
+      foreach($this_overboard['msgids'] as $key => $value) {
+        $target = $this_overboard['msgids'][$key];
+        if($target['date'] < (time() - (86400 * $article_age))) {
+          file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Expiring: ".$key, FILE_APPEND);
+          unset($this_overboard['threads'][$target['date']]);
+          unset($this_overboard['msgids'][$key]);
+          unset($this_overboard['threadlink'][$key]);
         }
       }
-      echo $result;
-      if($results++ > ($maxdisplay - 2))
-	    break;
+      $this_overboard['expire'] = time();
     }
+    if($prune) {
+      file_put_contents($cachefile, serialize($this_overboard));
+    }
+}
 
-    show_overboard_footer(null, $results, null);
-
-echo '</body></html>';
+function display_threads($threads, $oldest) {
+    global $thissite, $logfile, $config_name, $snippetlength, $maxdisplay, $prune, $this_overboard;
+    echo '<table cellspacing="0" width="100%" class="np_results_table">';
+    krsort($threads);
+    $results = 0;
+    foreach($threads as $key => $value) {
+      $target = $this_overboard['msgids'][$value];
+      if(!isset($target['msgid'])) {
+        $target = get_data_from_msgid($value);
+      }
+      if($target['date'] < $oldest) {
+          continue;
+      }
+      if($results > $maxdisplay) {
+        $prune = true;
+        unset($this_overboard['threads'][$target['date']]);
+        unset($this_overboard['threadlink'][$value]);
+        file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Pruning: ".$value, FILE_APPEND);
+      }
+      $article = get_db_data_from_msgid($target['msgid'], $target['newsgroup'], 1);
+        $poster = get_poster_name(mb_decode_mimeheader($target['name']));
+        $groupurl = $thissite."/thread.php?group="._rawurlencode($target['newsgroup']);
+        if(($results % 2) == 0){
+          echo '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
+        } else {
+          echo '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
+        }
+        $url = $thissite."/article-flat.php?id=".$target['number']."&group="._rawurlencode($target['newsgroup'])."#".$target['number'];
+        echo '<p class=np_ob_subject>';
+        echo '<b><a href="'.$url.'"><span>'.mb_decode_mimeheader($target['subject']).'</span></a></b>';
+        if(isset($this_overboard['threadlink'][$value])) {
+          $thread = get_data_from_msgid($this_overboard['threadlink'][$value]);
+          echo '<font class="np_ob_group"><a href="article-flat.php?id='.$thread[number].'&group='.rawurlencode($thread[newsgroup]).'#'.$thread[number].'"> (thread)</a></font>';
+        } 
+        echo '</p>';
+        echo '</p><p class=np_ob_group>';
+        echo '<a href="'.$groupurl.'"><span class="visited">'.$target['newsgroup'].'</span></a>';
+        echo '</p>';
+        echo '<p class=np_ob_posted_date>Posted: '.get_date_interval(date("D, j M Y H:i T",$target['date'])).' by: '.create_name_link($poster['name'], $poster['from']).'</p>';
+	    echo htmlentities(substr($article['search_snippet'], 0, $snippetlength));
+	    $results++;
+    }
+    echo "</table>";
+    return($results);
+}
 
 function show_overboard_header($grouplist) {
   global $text_thread, $text_article, $file_index, $file_thread, $user_time;
diff --git a/Rocksolid_Light/rslight/scripts/nocem.php b/Rocksolid_Light/rslight/scripts/nocem.php
index b9fe4e3..6337e96 100755
--- a/Rocksolid_Light/rslight/scripts/nocem.php
+++ b/Rocksolid_Light/rslight/scripts/nocem.php
@@ -145,25 +145,24 @@ function delete_message($messageid, $group) {
 
 function delete_message_from_overboard($config_name, $group, $messageid) {
   GLOBAL $spooldir;
-  $delkey = hash('crc32', serialize($messageid)); 
   $cachefile=$spooldir."/".$config_name."-overboard.dat";
   if(is_file($cachefile)) {
     $cached_overboard = unserialize(file_get_contents($cachefile));
-    if(isset($cached_overboard[$delkey])) {
-      unset($cached_overboard[$delkey]);
-      $stats = stat($cachefile);
+    if($target = $cached_overboard['msgids'][$messageid]) {
+      unset($cached_overboard['threads'][$target['date']]);
+      unset($cached_overboard['msgids'][$messageid]);
+      unset($cached_overboard['threadlink'][$messageid]);
       file_put_contents($cachefile, serialize($cached_overboard));
-      touch($cachefile, $stats[9]);
     }
   }
   $cachefile=$spooldir."/".$group."-overboard.dat";
   if(is_file($cachefile)) {
     $cached_overboard = unserialize(file_get_contents($cachefile));
-    if(isset($cached_overboard[$delkey])) {
-      unset($cached_overboard[$delkey]);
-      $stats = stat($cachefile);
+    if($target = $cached_overboard['msgids'][$messageid]) {
+      unset($cached_overboard['threads'][$target['date']]);
+      unset($cached_overboard['msgids'][$messageid]);
+      unset($cached_overboard['threadlink'][$messageid]);
       file_put_contents($cachefile, serialize($cached_overboard));
-      touch($cachefile, $stats[9]);
     }
   }
 }