автоматическая группировка, первый чих

src/api.js

@@ -25,4 +25,4 @@ ipc.on('get:files:find', function (e, data) {
 
 ipc.on('open:url', function (e, data) {
     require("openurl").open(data)
-});
+});

src/crawler.js

@@ -11,7 +11,9 @@ let cheerio = require('cheerio');
 let request = require('request-promise-native');
 let settings = require('./settings.js');
 let fileRepo = require('./dao.js').FileRepo;
-
+let groupRepo = require('./dao.js').GroupRepo;
+let string_similarity = require('string-similarity');
+let common_prefix = require('common-prefix');
 
 if (settings().get('crawl') === undefined) settings().set('crawl', {});
 
@@ -41,6 +43,64 @@ let defaults = {
     user: 0,
 };
 
+function doGrouping(rs, rj) {
+    let groupedFiles = new Set();
+    try {
+        [defaults.category.movies, defaults.category.ru_movies, defaults.category.series].forEach(cat => {
+            console.log(`grouping ${cat}`);
+            let oldGroups = Array.from(groupRepo.findAll());
+            let newGroups = [];
+            Array.from(oldGroups).forEach(g => Array.from(g.files).forEach(f => groupedFiles.add(f)));
+            let files = Array.from(_.sortByNat(Array.from(fileRepo.findAll()).filter(f => !groupedFiles.has(f.id)).filter(f => f.category === cat), f => f.name));
+            files.forEach((x, i, list) => x.lev0 = i === 0 ? 0.0 : string_similarity.compareTwoStrings(list[i - 1].name, x.name));
+            files.forEach((x, i, list) => x.lev1 = i === 0 ? 0.0 : string_similarity.compareTwoStrings(_.first(list[i - 1].name.split(" / ")), _.first(x.name.split(" / "))));
+            files.forEach(f => {
+                var lastGroup;
+                var nextGroup = function () {
+                    lastGroup = {realFiles: []};
+                    newGroups.push(lastGroup)
+                };
+                if (newGroups.length === 0) {
+                    nextGroup();
+                } else {
+                    lastGroup = _.last(newGroups);
+                    if (f.lev0 > 0.4 && f.lev1 > 0.85) {
+                        //do nothing
+                    } else {
+                        nextGroup();
+                    }
+                }
+                lastGroup.realFiles.push(f)
+            });
+            newGroups = newGroups.filter(g => g.realFiles.length > 1).map(g => {
+                g.name = common_prefix(g.realFiles.map(f => f.name)).trim();
+                g.files = g.realFiles.map(f => f.id);
+                return g
+            });
+            if (!_.isEmpty(newGroups)) {
+                let mergeGroups = Array.from(_.sortByNat(_.flatten([oldGroups, newGroups]), g => g.name));
+                mergeGroups.map((x, i, list) => {
+                    x.lev0 = i === 0 ? 0.0 : string_similarity.compareTwoStrings(list[i - 1].name, x.name);
+                    x.lev1 = i === 0 ? 0.0 : string_similarity.compareTwoStrings(_.first(list[i - 1].name.split(" / ")), _.first(x.name.split(" / ")));
+                    return x
+                }).forEach((x, i, list) => {
+                    if (x.lev0 > 0.4 && x.lev1 > 0.85 && i > 0) {
+                        list[i - 1].files = _.uniq(_.flatten(x.files, list[i - 1].files));
+                        x.files = [];
+                        if (_.has(x, "id")) list[i - 1] = x.id;
+                        x.id = undefined;
+                    }
+                });
+                groupRepo.save(mergeGroups.filter(g => !_.isEmpty(g.files)))
+            }
+        });
+        rs()
+    } catch (e) {
+        throw e;
+        rj(e)
+    }
+}
+
 let getUrlDay = function (date, cat, page) {
     if (_.isUndefined(page)) page = 0;
     return `${baseUrl}${page}/${cat}/${defaults.user}/${(defaults.order.date + defaults.sort.asc)};${moment(date).format('DD.MM.YYYY')};${moment(date).format('DD.MM.YYYY')}`;
@@ -163,7 +223,14 @@ function doAllCrawl() {
                 getCat(idx + 1);
             else {
                 console.log('all cats');
-                crawling = false;
+                new Promise(doGrouping).then(() => {
+                    console.log("all groups");
+                    crawling = false;
+                }, (e) => {
+                    console.log("grouping failed");
+                    console.error(e);
+                    crawling = false;
+                })
             }
         }, function (e) {
             crawling = false;

src/dao.js

@@ -64,6 +64,10 @@ class Repo {
     }
 }
 
+function _findAllDefault(table) {
+    return listResult(`SELECT * FROM ${table}`)
+}
+
 function _existsDefault(table, id) {
     if (_.isUndefined(id) || _.isNull(id)) return false;
     try {
@@ -81,6 +85,16 @@ function _getLastId() {
 }
 
 class GroupRepo extends Repo {
+
+    findAll() {
+        let groups = _findAllDefault('groups');
+        let files = _findAllDefault('filegroups');
+        return groups.map(g => {
+            g.files = files.filter(fg => fg.groupId == g.id).map(fg => fg.fileId);
+            return g;
+        })
+    }
+
     exists(id) {
         return _existsDefault('groups', id)
     }
@@ -127,6 +141,10 @@ class GroupRepo extends Repo {
 
 class FileRepo extends Repo {
 
+    findAll() {
+        return _findAllDefault('files')
+    }
+
     findFirstByOtherId(otherId) {
         return _.first(listResult(`SELECT * FROM files WHERE otherId = ${otherId}`))
     }

src/frontend.js

@@ -24,12 +24,12 @@ require('angular').module('UnderscoreStringFilter').filter('s', function () {
 require('angular').module('Ks').config(function ($mdAriaProvider) {
     $mdAriaProvider.disableWarnings();
 }).config(function ($mdThemingProvider) {
-    $mdThemingProvider
+    /*$mdThemingProvider
         .theme('default')
         .primaryPalette('green')
         .accentPalette('green')
         .warnPalette('orange')
-        .backgroundPalette('grey');
+        .backgroundPalette('grey');*/
 }).run(function (amMoment) {
     amMoment.changeLocale('ru');
     amMoment.changeTimezone("Europe/Moscow")

src/root.html

@@ -18,7 +18,7 @@
         text-decoration: underline;
     }
 </style>
-<div layout="column" class="main" ng-style="{'background': table.itemList.length == 0 ? 'url(img/stockvault-geometric-material-design-background241062.jpg)' : ''}">
+<div layout="column" class="main" ng-style="{'background': table.itemList.length == 0 ? 'url(img/material_design-wallpaper-1024.jpg)' : ''}">
     <toolbar></toolbar>
     <div layout="column">
         <div layout="row" flex="none" layout-align="start start" style="background: whitesmoke">
@@ -84,6 +84,11 @@
                             <md-icon ng-if="i.class == 'group' && !grExt[i.id]">expand_more</md-icon>
                         </td>
                         <!--<td md-cell>{{i.lev | number: 3}}</td>-->
+                        <td md-cell>
+                            <span ng-if="i.class == 'file'">
+                                <a ng-click="getFile(i, $event)" class="href"><md-icon>file_download</md-icon></a>
+                            </span>
+                        </td>
                         <td md-cell style="white-space: nowrap"><span am-time-ago="i.date"><md-tooltip>{{i.date | date : 'shortDate'}}</md-tooltip></span></td>
                         <td></td>
                     </tr>
@@ -93,6 +98,11 @@
                             <md-icon>link</md-icon>
                         </td>
                         <td md-cell ng-click="openFileUrl(i.id, i.otherId, $event)"><a ng-click="openFileUrl(f.id, f.otherId, $event)" class="href">{{f.name}}</a></td>
+                        <td md-cell>
+                            <a ng-click="getFile(f, $event)" class="href">
+                                <md-icon>file_download</md-icon>
+                            </a>
+                        </td>
                         <td md-cell style="white-space: nowrap"><span am-time-ago="f.date"><md-tooltip>{{f.date | date : 'shortDate'}}</md-tooltip></span></td>
                         <td></td>
                     </tr>

src/root.js

@@ -7,7 +7,7 @@ let string_similarity = require('string-similarity');
 require('../thirdparty/naturalSort');
 
 
-require('angular').module('Ks').constant('baseUrl', 'http://freedom-tor.org/torrent/').controller('RootController', function ($rootScope, $scope, $mdPanel, $timeout, baseUrl) {
+require('angular').module('Ks').constant('baseUrl', 'http://freedom-tor.org/torrent/').controller('RootController', function ($rootScope, $scope, $mdPanel, $timeout, $window, baseUrl) {
     $scope._ = _;
 
     let tabCats = {
@@ -82,6 +82,11 @@ require('angular').module('Ks').constant('baseUrl', 'http://freedom-tor.org/torr
         e.stopPropagation();
     };
 
+    $scope.getFile = function (file, e) {
+        ipc.send('open:url', file.magnet);
+        e.stopPropagation()
+    };
+
     $scope.selectTableRow = function (id) {
         let i = _.findIndex($scope.table.itemList, i => i.id == id);
         i++;

yarn.lock

@@ -3,12 +3,12 @@
 
 
 "@types/node@*":
-  version "9.4.6"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-9.4.6.tgz#d8176d864ee48753d053783e4e463aec86b8d82e"
+  version "9.4.7"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-9.4.7.tgz#57d81cd98719df2c9de118f2d5f3b1120dcd7275"
 
 "@types/node@^8.0.24":
-  version "8.9.4"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.9.4.tgz#dfd327582a06c114eb6e0441fa3d6fab35edad48"
+  version "8.9.5"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.9.5.tgz#162b864bc70be077e6db212b322754917929e976"
 
 abbrev@1:
   version "1.1.1"
@@ -49,8 +49,8 @@ angular-material-data-table@^0.10.10:
   resolved "https://registry.yarnpkg.com/angular-material-data-table/-/angular-material-data-table-0.10.10.tgz#229b402745ec1a146f07da8d1e526be6540849ee"
 
 angular-material@^1.1.4, angular-material@^1.1.7:
-  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/angular-material/-/angular-material-1.1.7.tgz#b3ebe28bc884495285932fb9cf4f58ed6ce6f4ab"
+  version "1.1.8"
+  resolved "https://registry.yarnpkg.com/angular-material/-/angular-material-1.1.8.tgz#6ddec2db18812506144ccf204cf8ecec330bc85c"
 
 angular-messages@^1.6.9:
   version "1.6.9"
@@ -89,8 +89,8 @@ array-find-index@^1.0.1:
   resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1"
 
 asar@^0.14.0:
-  version "0.14.2"
-  resolved "https://registry.yarnpkg.com/asar/-/asar-0.14.2.tgz#9ec5ec8dcb1904de288fd5ff94f2a8ed4014b094"
+  version "0.14.3"
+  resolved "https://registry.yarnpkg.com/asar/-/asar-0.14.3.tgz#c72a81542a48e3bca459fb1b07ee2b6adfae265d"
   dependencies:
     chromium-pickle-js "^0.2.0"
     commander "^2.9.0"
@@ -236,8 +236,12 @@ combined-stream@1.0.6, combined-stream@~1.0.5:
     delayed-stream "~1.0.0"
 
 commander@^2.9.0:
-  version "2.14.1"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa"
+  version "2.15.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f"
+
+common-prefix@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/common-prefix/-/common-prefix-1.1.0.tgz#e3a5ea7fafaefc7eb84e760523e1afb985f90f00"
 
 compare-version@^0.1.2:
   version "0.1.2"
@@ -404,8 +408,8 @@ electron-ipc-broadcast@^0.1.0:
   resolved "https://registry.yarnpkg.com/electron-ipc-broadcast/-/electron-ipc-broadcast-0.1.0.tgz#c686f0fee5c9f79b15231b99b4de95c447b4bc4a"
 
 electron-osx-sign@^0.4.1:
-  version "0.4.8"
-  resolved "https://registry.yarnpkg.com/electron-osx-sign/-/electron-osx-sign-0.4.8.tgz#f0b9fadded9e1e54ec35fa89877b5c6c34c7bc40"
+  version "0.4.10"
+  resolved "https://registry.yarnpkg.com/electron-osx-sign/-/electron-osx-sign-0.4.10.tgz#be4f3b89b2a75a1dc5f1e7249081ab2929ca3a26"
   dependencies:
     bluebird "^3.5.0"
     compare-version "^0.1.2"
@@ -415,8 +419,8 @@ electron-osx-sign@^0.4.1:
     plist "^2.1.0"
 
 electron-packager@^11.0.1:
-  version "11.0.1"
-  resolved "https://registry.yarnpkg.com/electron-packager/-/electron-packager-11.0.1.tgz#c2d1ff9eca8110be9ebc81826e2a921c04d1200e"
+  version "11.1.0"
+  resolved "https://registry.yarnpkg.com/electron-packager/-/electron-packager-11.1.0.tgz#90b5b41c952ccfc86f0bcb3a9c3fd30151978958"
   dependencies:
     asar "^0.14.0"
     debug "^3.0.0"
@@ -438,8 +442,8 @@ electron-packager@^11.0.1:
     yargs-parser "^9.0.2"
 
 electron@^1.8.2:
-  version "1.8.2"
-  resolved "https://registry.yarnpkg.com/electron/-/electron-1.8.2.tgz#a817cd733c2972b3c7cc4f777caf6e424b88014d"
+  version "1.8.4"
+  resolved "https://registry.yarnpkg.com/electron/-/electron-1.8.4.tgz#cca8d0e6889f238f55b414ad224f03e03b226a38"
   dependencies:
     "@types/node" "^8.0.24"
     electron-download "^3.0.1"
@@ -485,8 +489,8 @@ extsprintf@^1.2.0:
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f"
 
 fast-deep-equal@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff"
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz#c053477817c86b51daa853c81e059b733d023614"
 
 fast-json-stable-stringify@^2.0.0:
   version "2.0.0"
@@ -643,8 +647,8 @@ home-path@^1.0.1:
   resolved "https://registry.yarnpkg.com/home-path/-/home-path-1.0.5.tgz#788b29815b12d53bacf575648476e6f9041d133f"
 
 hosted-git-info@^2.1.4:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c"
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.6.0.tgz#23235b29ab230c576aab0d4f13fc046b0b038222"
 
 htmlparser2@^3.9.1:
   version "3.9.2"
@@ -887,9 +891,15 @@ mksnapshot@^0.3.0:
     fs-extra "0.26.7"
     request "^2.79.0"
 
-"moment@>=2.8.0 <3.0.0", moment@^2.20.1:
-  version "2.20.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.20.1.tgz#d6eb1a46cbcc14a2b2f9434112c1ff8907f313fd"
+moment-timezone@^0.5.14:
+  version "0.5.14"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.14.tgz#4eb38ff9538b80108ba467a458f3ed4268ccfcb1"
+  dependencies:
+    moment ">= 2.9.0"
+
+"moment@>= 2.9.0", "moment@>=2.8.0 <3.0.0", moment@^2.20.1:
+  version "2.21.0"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a"
 
 ms@2.0.0:
   version "2.0.0"
@@ -1122,8 +1132,8 @@ qs@~6.5.1:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8"
 
 rc@^1.1.2:
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd"
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.6.tgz#eb18989c6d4f4f162c399f79ddd29f3835568092"
   dependencies:
     deep-extend "~0.4.0"
     ini "~1.3.0"
@@ -1174,8 +1184,8 @@ readable-stream@^1.1.8, readable-stream@~1.1.9:
     string_decoder "~0.10.x"
 
 readable-stream@^2.0.2, readable-stream@^2.2.2:
-  version "2.3.4"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.4.tgz#c946c3f47fa7d8eabc0b6150f4a12f69a4574071"
+  version "2.3.5"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.5.tgz#b4f85003a938cbb6ecbce2a124fb1012bd1a838d"
   dependencies:
     core-util-is "~1.0.0"
     inherits "~2.0.3"
@@ -1213,8 +1223,8 @@ request-promise-native@^1.0.5:
     tough-cookie ">=2.3.3"
 
 request@^2.34, request@^2.45.0, request@^2.79.0:
-  version "2.83.0"
-  resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356"
+  version "2.85.0"
+  resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa"
   dependencies:
     aws-sign2 "~0.7.0"
     aws4 "^1.6.0"
@@ -1240,8 +1250,8 @@ request@^2.34, request@^2.45.0, request@^2.79.0:
     uuid "^3.1.0"
 
 resolve@^1.1.6:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36"
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.6.0.tgz#0fbd21278b27b4004481c395349e7aba60a9ff5c"
   dependencies:
     path-parse "^1.0.5"
 
@@ -1281,19 +1291,27 @@ sntp@2.x.x:
   dependencies:
     hoek "4.x.x"
 
-spdx-correct@~1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40"
+spdx-correct@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.0.0.tgz#05a5b4d7153a195bc92c3c425b69f3b2a9524c82"
   dependencies:
-    spdx-license-ids "^1.0.2"
+    spdx-expression-parse "^3.0.0"
+    spdx-license-ids "^3.0.0"
 
-spdx-expression-parse@~1.0.0:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz#9bdf2f20e1f40ed447fbe273266191fced51626c"
+spdx-exceptions@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz#2c7ae61056c714a5b9b9b2b2af7d311ef5c78fe9"
+
+spdx-expression-parse@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz#99e119b7a5da00e05491c9fa338b7904823b41d0"
+  dependencies:
+    spdx-exceptions "^2.1.0"
+    spdx-license-ids "^3.0.0"
 
-spdx-license-ids@^1.0.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz#c9df7a3424594ade6bd11900d596696dc06bac57"
+spdx-license-ids@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz#7a7cd28470cc6d3a1cfe6d66886f6bc430d3ac87"
 
 speedometer@~0.1.2:
   version "0.1.4"
@@ -1308,8 +1326,8 @@ sql.js@^0.5.0:
   resolved "https://registry.yarnpkg.com/sql.js/-/sql.js-0.5.0.tgz#f880dea18280a840e41df2209dc967422dfa8d81"
 
 sshpk@^1.7.0:
-  version "1.13.1"
-  resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3"
+  version "1.14.1"
+  resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.1.tgz#130f5975eddad963f1d56f92b9ac6c51fa9f83eb"
   dependencies:
     asn1 "~0.2.3"
     assert-plus "^1.0.0"
@@ -1428,8 +1446,8 @@ touch@0.0.3:
     nopt "~1.0.10"
 
 tough-cookie@>=2.3.3, tough-cookie@~2.3.3:
-  version "2.3.3"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
+  version "2.3.4"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.4.tgz#ec60cee38ac675063ffc97a5c18970578ee83655"
   dependencies:
     punycode "^1.4.1"
 
@@ -1489,11 +1507,11 @@ uuid@^3.1.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"
 
 validate-npm-package-license@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz#81643bcbef1bdfecd4623793dc4648948ba98338"
   dependencies:
-    spdx-correct "~1.0.0"
-    spdx-expression-parse "~1.0.0"
+    spdx-correct "^3.0.0"
+    spdx-expression-parse "^3.0.0"
 
 verror@1.10.0:
   version "1.10.0"