gnh1201/welsonjs
1
0
Fork 0
mirror of https://github.com/gnh1201/welsonjs.git synced 2025-05-08 20:56:04 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Update census.js

Browse Source
This commit is contained in:
Namhyeon Go 2022-08-29 03:01:11 +09:00 committed by GitHub
parent ae0cba0117
commit 8876f30837
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 66 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
examples/census.js
View File

@ -1,16 +1,18 @@
// Example: Crawling a title of websites
var FILE = require("lib/file"); var FILE = require("lib/file");
var HTTP = require("lib/http"); var HTTP = require("lib/http");
var Punycode = require("lib/punycode"); var Punycode = require("lib/punycode");
function main() function main() {
var lines = []; var lines = [];
var district = JSON.parse(FILE.readFile("data\\korea-administrative-district.json", "utf-8")); var district = JSON.parse(FILE.readFile("data\\korea-administrative-district.json", "utf-8"));
var districtData = district.data; var districtData = district.data;
var domains = splitLn(FILE.readFile("data\\domains.txt", "utf-8")); var domains = splitLn(FILE.readFile("data\\domains.txt", "utf-8"));
var digFrame = function(handler, domain, response) { var digFrame = function(handler, domain, response) {
var frameURLs = handler.getFrameURLs(); var frameURLs = handler.getFrameURLs();
if (frameURLs.length > 0) { if (frameURLs.length > 0) {
@ -26,9 +28,9 @@ var digFrame = function(handler, domain, response) {
} }
return response; return response;
}; };
domains.forEach(function(domain) { domains.forEach(function(domain) {
var handler = HTTP.create("CURL") var handler = HTTP.create("CURL")
.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36") .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36")
.setIsFollowRedirect(true) .setIsFollowRedirect(true)
@ -72,9 +74,9 @@ domains.forEach(function(domain) {
var bizNote = ''; var bizNote = '';
var row = [domain, title, bizNo, bizRegion, bizType, bizForm, bizNote]; var row = [domain, title, bizNo, bizRegion, bizType, bizForm, bizNote];
lines.push(row.join(":")); lines.push(row.join(":"));
}); });
FILE.appendFile("data\\matches.txt", lines.join("\r\n"), "utf-8"); FILE.appendFile("data\\matches.txt", lines.join("\r\n"), "utf-8");
} }
exports.main = main; exports.main = main;