From f5b04017748deda4420c52e61cc7c1bb2b9c68b9 Mon Sep 17 00:00:00 2001 From: Themroc <7hemroc@gmail.com> Date: Wed, 2 Feb 2022 17:21:17 +0100 Subject: [PATCH 1/6] Make sure we never run more than one instance --- Events.php | 11 +++++++++++ jobs/GetFeedUpdates.php | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/Events.php b/Events.php index e7235b4..c101cbc 100644 --- a/Events.php +++ b/Events.php @@ -11,6 +11,10 @@ class Events { + /** + * @var string mutex to acquire + */ + const MUTEX_ID = 'rss-cron'; /** * Defines what to do if admin menu is initialized. @@ -40,6 +44,11 @@ public static function onAdminMenuInit($event) */ public static function onCron($event) { + if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) { + Console::stdout("RSS cron execution skipped - already running!\n"); + return; + } + try { Console::stdout("Updating RSS news feeds...\n"); $ccmsEnabled = ContentContainerModuleState::find()-> @@ -62,6 +71,8 @@ public static function onCron($event) $event->sender->stderr($e->getMessage()."\n"); Yii::error($e); } + + Yii::$app->mutex->release(static::MUTEX_ID); } } diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index 40b5903..6f3f43e 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -49,6 +49,11 @@ class GetFeedUpdates extends ActiveJob private $newest; # newest date we are accepting private $items; # array of sij\humhub\modules\rss\components\RssElement keyed by pubDate + /** + * @var string mutex to acquire + */ + const MUTEX_ID = 'rss-queue'; + private function log($message) { if ( $this->logFileHandle ) { fwrite($this->logFileHandle, $message); @@ -444,6 +449,10 @@ private function downloadNewsFeed() */ public function run() { + if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) { + Console::stdout("RSS queue execution skipped - already running!\n"); + return; + } ####### $this->logFileHandle = fopen(dirname(__FILE__) . '/log.txt', 'w'); @@ -478,5 +487,6 @@ public function run() fclose($this->logFileHandle); } + Yii::$app->mutex->release(static::MUTEX_ID); } } From a377a0bec3980f788f37d5ef37a6964a38fa7f96 Mon Sep 17 00:00:00 2001 From: Themroc <7hemroc@gmail.com> Date: Wed, 2 Feb 2022 17:32:55 +0100 Subject: [PATCH 2/6] Make queue-run tell what it's doing --- jobs/GetFeedUpdates.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index 6f3f43e..5530a1d 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -3,6 +3,7 @@ namespace sij\humhub\modules\rss\jobs; use Yii; +use yii\helpers\Console; use humhub\modules\queue\ActiveJob; use humhub\modules\post\models\Post; @@ -107,6 +108,7 @@ private function postMessage($message, $datePublished = false) if ( count($oldContent) == 1 ) { $post = Post::findOne($oldContent[0]->object_id); $this->log("\n\n### update Post\n"); + Console::stdout("RSS queue: updating post... "); } } @@ -114,6 +116,7 @@ private function postMessage($message, $datePublished = false) if ( $post === null ) { $post = new Post($this->space); $this->log("\n\n### new Post\n"); + Console::stdout("RSS queue: creating new post... "); } $post->created_by = @@ -149,6 +152,7 @@ private function postMessage($message, $datePublished = false) ->query(); } + Console::stdout(Console::renderColoredString("%gdone.%n\n", 1)); } /** From 45500b5b87cf91a3ce5afbc2ed411340150b1189 Mon Sep 17 00:00:00 2001 From: Themroc <7hemroc@gmail.com> Date: Sat, 12 Feb 2022 06:05:15 +0100 Subject: [PATCH 3/6] Keep track of RSS item to Humhub post relationship --- jobs/GetFeedUpdates.php | 50 ++++++++++++++++++++------- migrations/m220202_180401_initial.php | 31 +++++++++++++++++ migrations/uninstall.php | 18 ++++++++++ models/RssPosts.php | 39 +++++++++++++++++++++ 4 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 migrations/m220202_180401_initial.php create mode 100644 migrations/uninstall.php create mode 100644 models/RssPosts.php diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index 5530a1d..4391882 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -12,6 +12,7 @@ use sij\humhub\modules\rss\components\MarkdownHelper; use sij\humhub\modules\rss\components\RssElement; use sij\humhub\modules\rss\controllers\RssController; +use sij\humhub\modules\rss\models\RssPosts; /** * Reads the RSS Feed URL for this space @@ -62,9 +63,9 @@ private function log($message) { } } -/** - * Creates a new Post in the Space - */ + /** + * Creates a new Post in the Space + */ private function postError($title, $info) { @@ -89,16 +90,23 @@ private function postError($title, $info) } -/** - * Creates a new Post in the Space - */ - private function postMessage($message, $datePublished = false) + /** + * Creates a new Post in the Space + */ + private function postMessage($message, $link = false, $datePublished = false) { - $post = null; + // find previous version of the post via db + if ( $link ) { + $url2id = RssPosts::findOne(['rss_link' => $link]); + if ( $url2id !== null ) + $post = Post::findOne($url2id->post_id); + } + // attempt to locate a previous version of the post - if ( $datePublished ) { + // guess this should go some day - themroc + if ( $post === null && $datePublished ) { $stamp = $datePublished->format("Y-m-d H:i:s"); $oldContent = Content::findAll([ 'contentcontainer_id' => $this->space->contentcontainer_id, @@ -107,8 +115,6 @@ private function postMessage($message, $datePublished = false) ]); if ( count($oldContent) == 1 ) { $post = Post::findOne($oldContent[0]->object_id); - $this->log("\n\n### update Post\n"); - Console::stdout("RSS queue: updating post... "); } } @@ -117,6 +123,19 @@ private function postMessage($message, $datePublished = false) $post = new Post($this->space); $this->log("\n\n### new Post\n"); Console::stdout("RSS queue: creating new post... "); + } else { + if ( $datePublished ) { + if ($stamp > $post->created_at) { + $this->log("\n\n### update Post\n"); + Console::stdout("RSS queue: updating post... "); + } else { + return; // not changed + } + } else { + if ( $post !== null ) { + return; // we assume it hasn't changed - better miss an update than rewrite the post every time. + } + } } $post->created_by = @@ -134,6 +153,13 @@ private function postMessage($message, $datePublished = false) $post->save(); $this->log(print_r($post, true)); + if (! $url2id ) { + $url2id = new RssPosts(); + $url2id->rss_link = $link; + $url2id->post_id = $post->id; + $url2id->save(); + } + // make it look like the space post was created at the same time as the RSS article // note $post->save() always sets the time stamps to "now" if ( $datePublished ) { @@ -259,7 +285,7 @@ private function parseNewsItem($item) } // post the message in the stream - $this->postMessage($message, $datePublished); + $this->postMessage($message, $link, $datePublished); } diff --git a/migrations/m220202_180401_initial.php b/migrations/m220202_180401_initial.php new file mode 100644 index 0000000..0a9f5c9 --- /dev/null +++ b/migrations/m220202_180401_initial.php @@ -0,0 +1,31 @@ +createTable('rss_posts', [ + 'id' => 'pk', + 'rss_link' => 'varchar(1024) NOT NULL', + 'post_id' => 'int(11) NOT NULL', + ], ''); + + // creates index for column `rss_link` + // hopefully mysql is smart enaugh to handle varchar(1024) decently + $this->createIndex( + 'idx-rss_posts-rss_link', + 'rss_posts', + 'rss_link', + ); + } + + public function down() + { + echo "m220202_180401_initial does not support migration down.\n"; + return false; + } + +} diff --git a/migrations/uninstall.php b/migrations/uninstall.php new file mode 100644 index 0000000..fa579d1 --- /dev/null +++ b/migrations/uninstall.php @@ -0,0 +1,18 @@ +dropTable('rss_posts'); + } + + public function down() + { + echo "uninstall does not support migration down.\n"; + return false; + } + +} diff --git a/models/RssPosts.php b/models/RssPosts.php new file mode 100644 index 0000000..8707a39 --- /dev/null +++ b/models/RssPosts.php @@ -0,0 +1,39 @@ + Date: Sun, 13 Feb 2022 05:21:22 +0100 Subject: [PATCH 4/6] Better console msgs, minor bugfix --- Events.php | 10 +++++++--- jobs/GetFeedUpdates.php | 32 ++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Events.php b/Events.php index c101cbc..8ead483 100644 --- a/Events.php +++ b/Events.php @@ -44,13 +44,16 @@ public static function onAdminMenuInit($event) */ public static function onCron($event) { + if ($event->sender->requestedRoute != "cron/run") + return; + if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) { Console::stdout("RSS cron execution skipped - already running!\n"); return; } try { - Console::stdout("Updating RSS news feeds...\n"); + Console::stdout("Queueing RSS feeds for spaces:"); $ccmsEnabled = ContentContainerModuleState::find()-> where(['module_id' => 'rss'])-> andWhere(['module_state' => 1])-> @@ -63,10 +66,11 @@ public static function onCron($event) if (! empty($lastrun) && time() < ($interval * 60 + $lastrun)) continue; $space->setSetting('lastrun', time(), 'rss'); - Console::stdout(" Queueing update for space \"" . $space->name . "\"\n"); +# Console::stdout(" Queueing update for space \"" . $space->name . "\"\n"); + Console::stdout(' "' . $space->name . '"'); Yii::$app->queue->push(new jobs\GetFeedUpdates(['space' => $space, 'force' => false])); } - Console::stdout(Console::renderColoredString("%gdone.%n\n", 1)); + Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1)); } catch (\Throwable $e) { $event->sender->stderr($e->getMessage()."\n"); Yii::error($e); diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index 4391882..caafd77 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -3,6 +3,7 @@ namespace sij\humhub\modules\rss\jobs; use Yii; +#use yii\BaseYii; use yii\helpers\Console; use humhub\modules\queue\ActiveJob; @@ -69,6 +70,9 @@ private function log($message) { private function postError($title, $info) { + Yii::warning("RSS Error: ".$title."\n".$info."\n", "RSS-Reader"); + return; + $post = new Post($this->space); $post->created_by = @@ -97,6 +101,10 @@ private function postMessage($message, $link = false, $datePublished = false) { $post = null; + if ( $datePublished ) { + $stamp = $datePublished->format("Y-m-d H:i:s"); + } + // find previous version of the post via db if ( $link ) { $url2id = RssPosts::findOne(['rss_link' => $link]); @@ -106,8 +114,7 @@ private function postMessage($message, $link = false, $datePublished = false) // attempt to locate a previous version of the post // guess this should go some day - themroc - if ( $post === null && $datePublished ) { - $stamp = $datePublished->format("Y-m-d H:i:s"); + if ( $post === null && $stamp ) { $oldContent = Content::findAll([ 'contentcontainer_id' => $this->space->contentcontainer_id, // 'created_by' => $this->created_by, // cant rely on this field if config changed @@ -122,19 +129,16 @@ private function postMessage($message, $link = false, $datePublished = false) if ( $post === null ) { $post = new Post($this->space); $this->log("\n\n### new Post\n"); - Console::stdout("RSS queue: creating new post... "); + Console::stdout("RSS queue: creating new post ".$link); } else { - if ( $datePublished ) { - if ($stamp > $post->created_at) { - $this->log("\n\n### update Post\n"); - Console::stdout("RSS queue: updating post... "); - } else { - return; // not changed - } + if ( ! $stamp ) { + return; // we assume it hasn't changed - better miss an update than rewrite the post every time. + } + if ($stamp > $post->created_at) { + $this->log("\n\n### update Post\n"); + Console::stdout("RSS queue: updating post ".$link); } else { - if ( $post !== null ) { - return; // we assume it hasn't changed - better miss an update than rewrite the post every time. - } + return; // not changed } } @@ -178,7 +182,7 @@ private function postMessage($message, $link = false, $datePublished = false) ->query(); } - Console::stdout(Console::renderColoredString("%gdone.%n\n", 1)); + Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1)); } /** From 85cf54174f766f629debc4ae99cabc7b0f495dd0 Mon Sep 17 00:00:00 2001 From: Themroc <7hemroc@gmail.com> Date: Sun, 13 Feb 2022 05:22:24 +0100 Subject: [PATCH 5/6] v0.1.3 --- module.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module.json b/module.json index 491a312..d2461c9 100644 --- a/module.json +++ b/module.json @@ -4,7 +4,7 @@ "description": "RSS News Reader for a HumHub Space.", "keywords": [ ], - "version": "0.1.2", + "version": "0.1.3", "humhub": { "minVersion": "1.2" }, From 43a0fcde5e125b9c6c6b4b8302a598a145c85e28 Mon Sep 17 00:00:00 2001 From: Themroc <7hemroc@gmail.com> Date: Sun, 13 Feb 2022 16:23:12 +0100 Subject: [PATCH 6/6] Report errors as errors rather than warnings --- jobs/GetFeedUpdates.php | 2 +- module.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index caafd77..faf45ee 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -70,7 +70,7 @@ private function log($message) { private function postError($title, $info) { - Yii::warning("RSS Error: ".$title."\n".$info."\n", "RSS-Reader"); + Yii::error("RSS-Reader: ".$title."\n".$info."\n", "RSS-Reader"); return; $post = new Post($this->space); diff --git a/module.json b/module.json index d2461c9..2ab31fe 100644 --- a/module.json +++ b/module.json @@ -4,7 +4,7 @@ "description": "RSS News Reader for a HumHub Space.", "keywords": [ ], - "version": "0.1.3", + "version": "0.1.4", "humhub": { "minVersion": "1.2" },