novabbs
/
rocksolid-light
3
0
Fork 1
Code Issues 22 Pull Requests Releases Wiki Activity

Overview flat file removed. Create history.db3 to track deleted articles.

This commit is contained in:
Retro_Guy 2023-08-16 09:41:13 -07:00
parent c9d0d1449e
commit 0983b18399
6 changed files with 130 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
Rocksolid_Light/rocksolid/auth.inc
View File

89
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -1460,6 +1460,37 @@ function mail_db_open($database, $table='messages') {
return($dbh); return($dbh);
} }
function history_db_open($database, $table='history') {
try {
$dbh = new PDO('sqlite:'.$database);
} catch (PDOException $e) {
echo 'Connection failed: '.$e->getMessage();
exit;
}
$dbh->exec("CREATE TABLE IF NOT EXISTS $table(
id INTEGER PRIMARY KEY,
newsgroup TEXT,
number TEXT,
msgid TEXT,
status TEXT,
statusdate TEXT,
statusreason TEXT,
statusnotes TEXT,
unique (newsgroup, msgid),
unique (newsgroup, number))");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_status on '.$table.'(status)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup on '.$table.'(newsgroup)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_msgid on '.$table.'(msgid)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup_number on '.$table.'(newsgroup,number)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_statusdate on '.$table.'(statusdate)');
$stmt->execute();
return($dbh);
}
function overview_db_open($database, $table='overview') { function overview_db_open($database, $table='overview') {
try { try {
$dbh = new PDO('sqlite:'.$database); $dbh = new PDO('sqlite:'.$database);
@ -1467,7 +1498,7 @@ function overview_db_open($database, $table='overview') {
echo 'Connection failed: '.$e->getMessage(); echo 'Connection failed: '.$e->getMessage();
exit; exit;
} }
$dbh->exec("CREATE TABLE IF NOT EXISTS overview( $dbh->exec("CREATE TABLE IF NOT EXISTS $table(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
newsgroup TEXT, newsgroup TEXT,
number TEXT, number TEXT,
@ -1481,27 +1512,27 @@ function overview_db_open($database, $table='overview') {
lines TEXT, lines TEXT,
xref TEXT, xref TEXT,
unique (newsgroup, msgid))"); unique (newsgroup, msgid))");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on '.$table.'(date)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup on overview(newsgroup)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup on '.$table.'(newsgroup)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_msgid on overview(msgid)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_msgid on '.$table.'(msgid)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup_number on overview(newsgroup,number)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup_number on '.$table.'(newsgroup,number)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_name on overview(name)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_name on '.$table.'(name)');
$stmt->execute(); $stmt->execute();
return($dbh); return($dbh);
} }
function article_db_open($database) { function article_db_open($database, $table='articles') {
try { try {
$dbh = new PDO('sqlite:'.$database); $dbh = new PDO('sqlite:'.$database);
} catch (PDOException $e) { } catch (PDOException $e) {
echo 'Connection failed: '.$e->getMessage(); echo 'Connection failed: '.$e->getMessage();
exit; exit;
} }
$dbh->exec("CREATE TABLE IF NOT EXISTS articles( $dbh->exec("CREATE TABLE IF NOT EXISTS $table(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
newsgroup TEXT, newsgroup TEXT,
number TEXT UNIQUE, number TEXT UNIQUE,
@ -1512,13 +1543,13 @@ function article_db_open($database) {
search_snippet TEXT, search_snippet TEXT,
article TEXT)"); article TEXT)");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_number on articles(number)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_number on '.$table.'(number)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_date on articles(date)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_date on '.$table.'(date)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_msgid on articles(msgid)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_msgid on '.$table.'(msgid)');
$stmt->execute(); $stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_name on articles(name)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_name on '.$table.'(name)');
$stmt->execute(); $stmt->execute();
$dbh->exec("CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5( $dbh->exec("CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5(
@ -1529,10 +1560,10 @@ function article_db_open($database) {
name, name,
subject, subject,
search_snippet)"); search_snippet)");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_insert AFTER INSERT ON articles BEGIN $dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_insert AFTER INSERT ON $table BEGIN
INSERT INTO search_fts(newsgroup, number, msgid, date, name, subject, search_snippet) VALUES(new.newsgroup, new.number, new.msgid, new.date, new.name, new.subject, new.search_snippet); INSERT INTO search_fts(newsgroup, number, msgid, date, name, subject, search_snippet) VALUES(new.newsgroup, new.number, new.msgid, new.date, new.name, new.subject, new.search_snippet);
END;"); END;");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_delete AFTER DELETE ON articles BEGIN $dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_delete AFTER DELETE ON $table BEGIN
DELETE FROM search_fts WHERE msgid = old.msgid; DELETE FROM search_fts WHERE msgid = old.msgid;
END;"); END;");
return($dbh); return($dbh);
@ -1699,6 +1730,36 @@ function verify_gpg_signature($res, $signed_text) {
} }
} }
function is_deleted_post($group, $number) {
global $spooldir;
$database = $spooldir.'/history.db3';
$table = 'history';
$dbh = history_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup AND number=:nicole");
$stmt->bindParam(':newsgroup', $group);
$stmt->bindParam(':nicole', $number);
$stmt->execute();
$status = false;
while($row = $stmt->fetch()) {
if($row['status'] == "deleted") {
$status = "430 Article Deleted";
break;
}
}
$dbh = null;
return $status;
}
function add_to_history($group, $number, $msgid, $status, $statusdate, $statusreason=null, $statusnotes=null) {
global $spooldir;
$history = $spooldir.'/history.db3';
$history_dbh = history_db_open($history);
$history_sql = 'INSERT OR IGNORE INTO history(newsgroup, number, msgid, status, statusdate, statusreason, statusnotes) VALUES(?,?,?,?,?,?,?)';
$history_stmt = $history_dbh->prepare($history_sql);
$history_stmt->execute([$group, $number, $msgid, $status, $statusdate, $statusreason, $statusnotes]);
$history_dbh = null;
}
function get_db_data_from_msgid($msgid, $group) { function get_db_data_from_msgid($msgid, $group) {
global $spooldir; global $spooldir;
$database = $spooldir.'/'.$group.'-articles.db3'; $database = $spooldir.'/'.$group.'-articles.db3';

49
Rocksolid_Light/rslight/scripts/expire.php
View File

@ -42,15 +42,6 @@
echo "Expire $group articles before $showme\n"; echo "Expire $group articles before $showme\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring: articles before ".$showme, FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring: articles before ".$showme, FILE_APPEND);
echo "Expiring overview database...\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring overview database...", FILE_APPEND);
$database = $spooldir.'/articles-overview.db3';
$dbh = overview_db_open($database);
$query = $dbh->prepare('DELETE FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
$query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$dbh = null;
if($CONFIG['article_database'] == '1') { if($CONFIG['article_database'] == '1') {
echo "Expiring article database...\n"; echo "Expiring article database...\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring article database...", FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring article database...", FILE_APPEND);
@ -61,21 +52,31 @@ file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group."
$articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]); $articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$articles_dbh = null; $articles_dbh = null;
} }
} else { // Expire tradspool and remove from newsportal }
$database = $spooldir.'/articles-overview.db3'; // Expire tradspool and remove from newsportal
$dbh = overview_db_open($database); echo "Expiring overview database and writing history...\n";
$query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND date<:expireme'); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring overview database and writing history...", FILE_APPEND);
$query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$grouppath = preg_replace('/\./', '/', $group); $database = $spooldir.'/articles-overview.db3';
while($row = $query->fetch()) { $dbh = overview_db_open($database);
if(is_file($spooldir.'/articles/'.$grouppath.'/'.$row['number'])) { $query = $dbh->prepare('SELECT * FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']); $query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
} $stmt = $articles_dbh->prepare('DELETE FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
thread_cache_removearticle($group,$row['number']); $grouppath = preg_replace('/\./', '/', $group);
$status = "deleted";
$statusdate = time();
$statusreason = "expired";
while($row = $query->fetch()) {
if(is_file($spooldir.'/articles/'.$grouppath.'/'.$row['number'])) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
} }
$dbh = null; add_to_history($group, $row['number'], $row['msgid'], $status, $statusdate, $statusreason, $statusnotes);
thread_cache_removearticle($group,$row['number']);
} }
} $stmt->execute([':newsgroup' => $group, ':expireme' => $expireme]);
unlink($lockfile); $dbh = null;
touch($spooldir.'/'.$config_name.'-expire-timer'); unlink($lockfile);
touch($spooldir.'/'.$config_name.'-expire-timer');
echo "Expired ".$i." articles for ".$group."\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Expired ".$i." articles for ".$group, FILE_APPEND);
?> ?>

43
Rocksolid_Light/rslight/scripts/nocem.php
View File

@ -90,14 +90,7 @@ function delete_message($messageid, $group) {
} }
} }
} }
if($config_name) {
$database = $spooldir.'/articles-overview.db3';
$dbh = overview_db_open($database);
$query = $dbh->prepare('DELETE FROM overview WHERE msgid=:messageid');
$query->execute(['messageid' => $messageid]);
$dbh = null;
// thread_cache_removearticle($group,$messageid);
}
if($CONFIG['article_database'] == '1') { if($CONFIG['article_database'] == '1') {
$database = $spooldir.'/'.$group.'-articles.db3'; $database = $spooldir.'/'.$group.'-articles.db3';
if(is_file($database)) { if(is_file($database)) {
@ -107,20 +100,28 @@ function delete_message($messageid, $group) {
$articles_dbh = null; $articles_dbh = null;
} }
} }
// Tradspool // Handle overview and history
if($CONFIG['article_database'] != '1') { $database = $spooldir.'/articles-overview.db3';
$database = $spooldir.'/articles-overview.db3'; $dbh = overview_db_open($database);
$dbh = overview_db_open($database); $stmt_del = $dbh->prepare('DELETE FROM overview WHERE newsgroup=:newsgroup AND msgid=:msgid');
$query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND msgid<:msgid'); $query = $dbh->prepare('SELECT * FROM overview WHERE newsgroup=:newsgroup AND msgid=:msgid');
$query->execute([':newsgroup' => $group, ':msgid' => $messageid]); $query->execute([':newsgroup' => $group, ':msgid' => $messageid]);
$grouppath = preg_replace('/\./', '/', $group); $grouppath = preg_replace('/\./', '/', $group);
while($row = $query->fetch()) { $status = "deleted";
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']); $statusdate = time();
} $statusreason = "nocem";
$dbh = null; $statusnotes = null;
while($row = $query->fetch()) {
if(is_file($spooldir.'/articles/'.$grouppath.'/'.$row['number'])) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
} }
delete_message_from_overboard($config_name, $group, $messageid); delete_message_from_overboard($config_name, $group, $messageid);
add_to_history($group, $row['number'], $row['msgid'], $status, $statusdate, $statusreason, $statusnotes);
thread_cache_removearticle($group, $row['number']);
}
$stmt_del->execute([':newsgroup' => $group, ':msgid' => $messageid]);
$dbh = null;
return; return;
} }

7
Rocksolid_Light/rslight/scripts/rslight-lib.php
View File

@ -250,9 +250,9 @@ set_time_limit(0);
} }
} }
function prepare_post($filename) { function prepare_post($filename) {
global $logdir, $spooldir; global $logdir, $spooldir, $config_dir, $rslight_gpg;
$logfile = $logdir.'/nntp.log'; $logfile = $logdir.'/nntp.log';
$message = file($filename, FILE_IGNORE_NEW_LINES); $message = file($filename, FILE_IGNORE_NEW_LINES);
$lines = 0; $lines = 0;
@ -1105,6 +1105,9 @@ $date_i,$mid_i,$references_i,$bytes_i,$lines_i,$xref_i,$body) {
if($local < 1) { if($local < 1) {
$local = 1; $local = 1;
} }
while(is_deleted_post($nntp_group, $local)) {
$local++;
}
if($article_date > time()) if($article_date > time())
$article_date = time(); $article_date = time();
$in_file=fopen($filename, 'r'); $in_file=fopen($filename, 'r');

3
Rocksolid_Light/rslight/scripts/spoolnews.php
View File

@ -186,6 +186,9 @@ function get_articles($ns, $group) {
if($local < 1) { if($local < 1) {
$local = 1; $local = 1;
} }
while(is_deleted_post($group, $local)) {
$local++;
}
} }
# Split group response line to get last article number # Split group response line to get last article number
$detail = explode(" ", $response); $detail = explode(" ", $response);