Merge branch 'master' of http://q.ocsf.in:9095/pk/kisybi

build.sh

@@ -4,4 +4,8 @@ docker build -t ocsf.in/kisybi .
 
 docker rm -f kisybi || true
 
-docker run -d --restart always --name kisybi -v /var/docker/kisybi/config.json:/home/node/kisybi/config.json -v /mnt/share/local/up/t/:/home/node/kisybi/tmp/done ocsf.in/kisybi
+sleep 10
+
+docker run -d --restart always --name kisybi -v /var/docker/kisybi/config.json:/home/node/kisybi/config.json -v /mnt/share/local/up/t/:/home/node/kisybi/tmp/done ocsf.in/kisybi
+
+docker logs kisybi

server.js

@@ -15,7 +15,7 @@ const ipb = require('./config.server.js').ipb
 let crawlState = 'off';
 
 let crawlFn = async () => {
-    ipb('global:refresh')
+    ipb('global:refresh:force')
 }
 
 ipc.on("global:refresh:done", () => {
@@ -48,7 +48,7 @@ let crawlFnSafe = async () => {
 }
 
 
-schedule.scheduleJob('0 3 * * *', () => {
+schedule.scheduleJob('50 23 * * *', () => {
     if(crawlState === 'off'){
         return new Promise(async () => await crawlFnSafe())
     } else {
@@ -57,5 +57,5 @@ schedule.scheduleJob('0 3 * * *', () => {
 });
 
 schedule.scheduleJob('*/30 * * * *', () => {
-    console.log("service is alive")
+    console.log(`service is alive ${new Date()}`)
 })

src/crawler.js

@@ -252,15 +252,31 @@ function doCrawl(cat, force) {
 
 var crawling = false;
 
-function doAllCrawl() {
+function doAllCrawl(force) {
     if (crawling) return;
+    force = !!force
     crawling = true;
 
     let cats = _.values(defaults.category);
 
     function getCat(idx) {
-        console.log(`get cat ${cats[idx]}`);
-        doCrawl(cats[idx]).then(function () {
+        let cat = cats[idx];
+        console.log(`get cat ${cat}`);
+
+        if(force){
+            var lastDates = settings().get('crawl').lastDates;
+            if (_.isUndefined(lastDates)) lastDates = {};
+            var lastDate = _.has(lastDates, cat) ? lastDates[cat] : '2024-09-01';
+            let date = moment(lastDate, 'YYYY-MM-DD').endOf('day');
+            if (date.isBefore(moment())) date.add(1, 'day');
+            if (date.isAfter(moment())) {  //rollback only last day if force
+                date.subtract(2, 'day');
+                lastDates[cat] = moment(date).format('YYYY-MM-DD');
+                settings().set('crawl', {lastDates: lastDates});
+            }
+        }
+
+        doCrawl(cat, force).then(function () {
             if (idx + 1 < cats.length)
                 getCat(idx + 1);
             else {
@@ -292,4 +308,8 @@ ipc.on('global:refresh', function () {
     if (!crawling) doAllCrawl();
 });
 
+ipc.on('global:refresh:force', function () {
+    if (!crawling) doAllCrawl('force');
+});
+
 module.exports = {};