разборки с дебагом электрона + разбор

backend.js

@@ -1,3 +1,4 @@
 require('./settings.js');
 require('./db.js');
 require('./ui.js');
+require('./crawler.js');

cli.js

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+
+const electron = require('electron');
+const S = require('underscore.string');
+
+const proc = require('child_process');
+
+const child = proc.spawn(electron, ['.', '--inspect-brk=' + '14888'], {stdio: 'inherit'});
+
+console.log(process.argv);
+
+child.on('close', function (code) {
+    process.exit(code)
+});

crawler.js

@@ -0,0 +1,67 @@
+let moment = require('moment');
+let _ = require('underscore');
+const baseUrl = 'http://freedom-tor.org/browse/';
+
+let settings = require('./settings.js');
+let cheerio = require('cheerio');
+let request = require('request-promise-native');
+
+if (settings().get('crawl') === undefined) settings().set('crawl', {});
+
+let defaults = {
+    category: {
+        all: 0,
+        movies: 1,
+    },
+    sort: {
+        asc: 1,
+        desc: 0
+    },
+    order: {
+        date: 0,
+        seed: 2,
+        peer: 4,
+        name: 6,
+        size: 8
+    },
+    user: 0,
+};
+
+let getUrlDay = function (date, cat, page) {
+    if (_.isUndefined(page)) page = 0;
+    return `${baseUrl}${page}/${cat}/${defaults.user}/${(defaults.order.date + defaults.sort.asc)};${moment(date).format('DD.MM.YYYY')};${moment(date).format('DD.MM.YYYY')}`;
+};
+
+function doCrawl(force) {
+    lastDate = settings().get('crawl').lastDate;
+    if (_.isUndefined(lastDate)) lastDate = '2017-12-31';
+    let date = moment(lastDate, 'YYYY-MM-DD').add(1, 'day');
+    if (date.isAfter(moment.now()) && !force) return;
+    let nextUrl = getUrlDay(date.toDate(), defaults.category.movies);
+    console.log(nextUrl);
+    request({
+        uri: nextUrl,
+        transform: function (body) {
+            return cheerio.load(body);
+        }
+    }).then($ => {
+        let tbody = $('#index').first()[0].children[0].children[0];
+        tbody.children.slice(1).map(r => r.children[1].children).map(al => {
+            return {
+                dl: al[0].attribs['href'],
+                magnet: al[1].attribs['href'],
+                name: al[3].children[0].data
+            }
+        }).forEach(l => console.log(l))
+    }, e => {
+        throw e;
+    });
+    //settings().set('crawl', {lastDate: moment(date).format('DD-MM-YYYY')});
+    //setTimeout(doCrawl, 1000);
+}
+
+setTimeout(function () {
+    doCrawl(true)
+}, 1000);
+
+module.exports = {};

db.js

@@ -8,7 +8,7 @@ function firstRun(db) {
 }
 
 function initDb() {
-    let filepath = path.join(settings.userPath, settings.userDbFile);
+    let filepath = path.join(settings().userPath, settings().userDbFile);
     if (!fs.existsSync(filepath)) {
         let db = new SQL.Database();
         firstRun(db);
@@ -17,12 +17,12 @@ function initDb() {
 }
 
 function loadDb() {
-    let filepath = path.join(settings.userPath, settings.userDbFile);
+    let filepath = path.join(settings().userPath, settings().userDbFile);
     return new SQL.Database(fs.readFileSync(filepath));
 }
 
 function saveDb(db) {
-    let filepath = path.join(settings.userPath, settings.userDbFile);
+    let filepath = path.join(settings().userPath, settings().userDbFile);
     fs.writeFileSync(filepath, new Buffer(db.export()));
 }
 

package-lock.json

@@ -10,9 +10,9 @@
       "integrity": "sha512-ZxrhcBxlZA7tn0HFf7ebUYfR9aHyBgjyavBLzyrYMYuAMbONBPY4S5O35562iV2FfwnZCjQky3gTDy1B3jSZ2Q=="
     },
     "ajv": {
-      "version": "5.5.1",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.1.tgz",
-      "integrity": "sha1-s4u4h22ehr7plJVqBOch6IskjrI=",
+      "version": "5.5.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
+      "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
       "requires": {
         "co": "4.6.0",
         "fast-deep-equal": "1.0.0",
@@ -107,12 +107,17 @@
         "tweetnacl": "0.14.5"
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "boom": {
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
       "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=",
       "requires": {
-        "hoek": "4.2.0"
+        "hoek": "4.2.1"
       }
     },
     "brace-expansion": {
@@ -148,6 +153,19 @@
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
+    "cheerio": {
+      "version": "1.0.0-rc.2",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
+      "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=",
+      "requires": {
+        "css-select": "1.2.0",
+        "dom-serializer": "0.1.0",
+        "entities": "1.1.1",
+        "htmlparser2": "3.9.2",
+        "lodash": "4.17.5",
+        "parse5": "3.0.3"
+      }
+    },
     "co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -159,9 +177,9 @@
       "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
     },
     "combined-stream": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
-      "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=",
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz",
+      "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=",
       "requires": {
         "delayed-stream": "1.0.0"
       }
@@ -228,11 +246,27 @@
           "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz",
           "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==",
           "requires": {
-            "hoek": "4.2.0"
+            "hoek": "4.2.1"
           }
         }
       }
     },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "1.0.0",
+        "css-what": "2.1.0",
+        "domutils": "1.5.1",
+        "nth-check": "1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz",
+      "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0="
+    },
     "currently-unhandled": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz",
@@ -272,6 +306,44 @@
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
     },
+    "dom-serializer": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz",
+      "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=",
+      "requires": {
+        "domelementtype": "1.1.3",
+        "entities": "1.1.1"
+      },
+      "dependencies": {
+        "domelementtype": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs="
+        }
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz",
+      "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI="
+    },
+    "domhandler": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz",
+      "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=",
+      "requires": {
+        "domelementtype": "1.3.0"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0.1.0",
+        "domelementtype": "1.3.0"
+      }
+    },
     "ecc-jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
@@ -307,6 +379,11 @@
         "sumchecker": "1.3.1"
       }
     },
+    "entities": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
+      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
+    },
     "error-ex": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz",
@@ -374,13 +451,13 @@
       "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
     },
     "form-data": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz",
-      "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=",
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz",
+      "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=",
       "requires": {
         "asynckit": "0.4.0",
-        "combined-stream": "1.0.5",
-        "mime-types": "2.1.17"
+        "combined-stream": "1.0.6",
+        "mime-types": "2.1.18"
       }
     },
     "fs-extra": {
@@ -441,7 +518,7 @@
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
       "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
       "requires": {
-        "ajv": "5.5.1",
+        "ajv": "5.5.2",
         "har-schema": "2.0.0"
       }
     },
@@ -452,14 +529,14 @@
       "requires": {
         "boom": "4.3.1",
         "cryptiles": "3.1.2",
-        "hoek": "4.2.0",
+        "hoek": "4.2.1",
         "sntp": "2.1.0"
       }
     },
     "hoek": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz",
-      "integrity": "sha512-v0XCLxICi9nPfYrS9RL8HbYnXi9obYAeLbSP00BmnZwCK9+Ih9WOjoZ8YoHCoav2csqn4FOz4Orldsy2dmDwmQ=="
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz",
+      "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA=="
     },
     "home-path": {
       "version": "1.0.5",
@@ -471,6 +548,53 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.5.0.tgz",
       "integrity": "sha512-pNgbURSuab90KbTqvRPsseaTxOJCZBD0a7t+haSN33piP9cCM4l0CqdzAif2hUqm716UovKB2ROmiabGAKVXyg=="
     },
+    "htmlparser2": {
+      "version": "3.9.2",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz",
+      "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=",
+      "requires": {
+        "domelementtype": "1.3.0",
+        "domhandler": "2.4.1",
+        "domutils": "1.5.1",
+        "entities": "1.1.1",
+        "inherits": "2.0.3",
+        "readable-stream": "2.3.4"
+      },
+      "dependencies": {
+        "isarray": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+        },
+        "process-nextick-args": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+          "integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o="
+        },
+        "readable-stream": {
+          "version": "2.3.4",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.4.tgz",
+          "integrity": "sha1-yUbD9H+n2Oq8C2FQ9KEvaaRXQHE=",
+          "requires": {
+            "core-util-is": "1.0.2",
+            "inherits": "2.0.3",
+            "isarray": "1.0.0",
+            "process-nextick-args": "2.0.0",
+            "safe-buffer": "5.1.1",
+            "string_decoder": "1.0.3",
+            "util-deprecate": "1.0.2"
+          }
+        },
+        "string_decoder": {
+          "version": "1.0.3",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
+          "integrity": "sha1-D8Z9fBQYJd6UKC3VNr7GubzoYKs=",
+          "requires": {
+            "safe-buffer": "5.1.1"
+          }
+        }
+      }
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -617,6 +741,11 @@
         "strip-bom": "2.0.0"
       }
     },
+    "lodash": {
+      "version": "4.17.5",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz",
+      "integrity": "sha1-maktZcAnLevoyWtgV7yPv6O+1RE="
+    },
     "loud-rejection": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz",
@@ -649,16 +778,16 @@
       }
     },
     "mime-db": {
-      "version": "1.30.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz",
-      "integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE="
+      "version": "1.33.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
+      "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ=="
     },
     "mime-types": {
-      "version": "2.1.17",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz",
-      "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=",
+      "version": "2.1.18",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz",
+      "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==",
       "requires": {
-        "mime-db": "1.30.0"
+        "mime-db": "1.33.0"
       }
     },
     "minimatch": {
@@ -689,6 +818,11 @@
         }
       }
     },
+    "moment": {
+      "version": "2.20.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.20.1.tgz",
+      "integrity": "sha512-Yh9y73JRljxW5QxN08Fner68eFLxM5ynNOAw2LbIB1YAGeQzZT8QFSUvkAz609Zf+IHhhaUxqZK8dG3W/+HEvg=="
+    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -705,6 +839,14 @@
         "validate-npm-package-license": "3.0.1"
       }
     },
+    "nth-check": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz",
+      "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=",
+      "requires": {
+        "boolbase": "1.0.0"
+      }
+    },
     "nugget": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz",
@@ -755,6 +897,14 @@
         "error-ex": "1.3.1"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha1-BC95L/3TaFFVHPTp4Gazh0q0W1w=",
+      "requires": {
+        "@types/node": "8.0.57"
+      }
+    },
     "path-exists": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
@@ -905,17 +1055,17 @@
         "aws-sign2": "0.7.0",
         "aws4": "1.6.0",
         "caseless": "0.12.0",
-        "combined-stream": "1.0.5",
+        "combined-stream": "1.0.6",
         "extend": "3.0.1",
         "forever-agent": "0.6.1",
-        "form-data": "2.3.1",
+        "form-data": "2.3.2",
         "har-validator": "5.0.3",
         "hawk": "6.0.2",
         "http-signature": "1.2.0",
         "is-typedarray": "1.0.0",
         "isstream": "0.1.2",
         "json-stringify-safe": "5.0.1",
-        "mime-types": "2.1.17",
+        "mime-types": "2.1.18",
         "oauth-sign": "0.8.2",
         "performance-now": "2.1.0",
         "qs": "6.5.1",
@@ -923,7 +1073,7 @@
         "stringstream": "0.0.5",
         "tough-cookie": "2.3.3",
         "tunnel-agent": "0.6.0",
-        "uuid": "3.1.0"
+        "uuid": "3.2.1"
       }
     },
     "rimraf": {
@@ -962,7 +1112,7 @@
       "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz",
       "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==",
       "requires": {
-        "hoek": "4.2.0"
+        "hoek": "4.2.1"
       }
     },
     "spdx-correct": {
@@ -988,6 +1138,11 @@
       "resolved": "https://registry.npmjs.org/speedometer/-/speedometer-0.1.4.tgz",
       "integrity": "sha1-mHbb0qFp0xFUAtSObqYynIgWpQ0="
     },
+    "sprintf-js": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.1.tgz",
+      "integrity": "sha1-Nr54Mgr+WAH2zqPueLblqrlA6gw="
+    },
     "sql.js": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-0.5.0.tgz",
@@ -1117,15 +1272,24 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
       "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI="
     },
+    "underscore.string": {
+      "version": "3.3.4",
+      "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-3.3.4.tgz",
+      "integrity": "sha1-LCo/n4PmR2L9xF5s6sZRQoZCE9s=",
+      "requires": {
+        "sprintf-js": "1.1.1",
+        "util-deprecate": "1.0.2"
+      }
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
     },
     "uuid": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
-      "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
+      "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
     },
     "validate-npm-package-license": {
       "version": "3.0.1",

package.json

@@ -14,8 +14,13 @@
     "angular-messages": "^1.6.7",
     "angular-sanitize": "^1.6.7",
     "angular-ui-router": "^0.4.3",
+    "cheerio": "^1.0.0-rc.2",
     "electron": "^1.8.1",
+    "moment": "^2.20.1",
+    "request": "latest",
+    "request-promise-native": "^1.0.5",
     "sql.js": "^0.5.0",
-    "underscore": "^1.8.3"
+    "underscore": "^1.8.3",
+    "underscore.string": "^3.3.4"
   }
 }

settings.js

@@ -37,4 +37,12 @@ initConfig();
 loadConfig();
 saveConfig();
 
-module.exports = settings;
+module.exports = () => _.extend({
+    set: function (k, v) {
+        settings.userConfig[k] = this.userConfig[k] = v;
+        saveConfig();
+    },
+    get: function (k) {
+        return settings.userConfig[k];
+    }
+}, settings);