diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..c5d0721
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+a.out
+allowed_ips.txt
+allowed_patterns.txt
+patterns.txt
\ No newline at end of file
diff --git a/allowed_ips.txt b/allowed_ips.txt
deleted file mode 100644
index 849702f..0000000
--- a/allowed_ips.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-109.128.185.4
-192.168.
-127.
-10.
diff --git a/allowed_patterns.txt b/allowed_patterns.txt
deleted file mode 100644
index fe74fac..0000000
--- a/allowed_patterns.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-GET /server.git/
-GET /matrix/thirdparty/nheko.git/
- /.well-known/matrix/
- /_matrix/client/
- /_matrix/federation/
- /_matrix/key
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..5ae5300
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,26 @@
+{
+ "nodes": {
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1748615678,
+ "narHash": "sha256-gRB3O1QqBm8RZPqG9KjmmIh7iavYEPTMpDYInDYfyjI=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "c579bdd6d57b214c6b89c46b5b15e22b83ca7f9e",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "nixpkgs": "nixpkgs"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..c384ad5
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,11 @@
+{
+ inputs.nixpkgs.url = "github:NixOS/nixpkgs";
+
+ outputs = { self, nixpkgs }: {
+ devShells.default = nixpkgs.lib.mkShell {
+ buildInputs = [
+ nixpkgs.libnftnl
+ ];
+ };
+ };
+}
diff --git a/main.c b/main.c
index 18e9f80..9ab4181 100644
--- a/main.c
+++ b/main.c
@@ -29,6 +29,10 @@ bool print_allowed_pattern = false;
bool print_already_banned_ip = false;
bool print_indeterminate = false;
bool print_bans = true;
+bool print_only = false;
+bool print_progress = true;
+
+bool dump_patterns = false;
char* allowed_patterns_path = ALLOWED_PATTERNS_FILE;
char* allowed_ips_path = ALLOWED_IPS_FILE;
@@ -85,6 +89,7 @@ void read_lines_cb(const char* filename, void cb(char*, void*), void* cbData)
int count = 0;
char* buf = xmalloc(10000);
+ clock_t times[1000];
while (fgets(buf, 10000, file) != NULL)
{
if (buf[0] == '#')
@@ -95,17 +100,17 @@ void read_lines_cb(const char* filename, void cb(char*, void*), void* cbData)
buf[strcspn(buf, "\n")] = 0;
char* line = strdup(buf);
- if (cb != nullptr)
- cb(line, cbData);
+ cb(line, cbData);
free(line);
- if (count % 10 == 0)
+ times[count % 1000] = clock();
+ if (print_progress && count % 10 == 0)
{
double elapsed = (clock() - lastPrint) * 1000.0 / CLOCKS_PER_SEC;
+ double totalElapsed = (clock() - ctime) * 1000.0 / CLOCKS_PER_SEC;
if (elapsed > 50.0)
{
- fprintf(stderr, "\rRead %d lines from %s in %.3f ms (%.3f/s)\r", count, filename, (clock() - ctime) * 1000.0 / CLOCKS_PER_SEC,
- count * 1000.0 / (clock() - ctime));
+ fprintf(stderr, "\rRead %d lines from %s in %.3f ms (%.3f/s)\r", count, filename, totalElapsed, count / (totalElapsed / 1000.0));
lastPrint = clock();
}
}
@@ -114,7 +119,7 @@ void read_lines_cb(const char* filename, void cb(char*, void*), void* cbData)
free(buf);
fclose(file);
- printf("Read %d definitions from %s in %.3f ms (%.3f/s)\n", count, filename, (clock() - ctime) * 1000.0 / CLOCKS_PER_SEC, count * 1000.0 / (clock() - ctime));
+ fprintf(stderr, "Read %d definitions from %s in %.3f ms (%.3f/s)\n", count, filename, (clock() - ctime) * 1000.0 / CLOCKS_PER_SEC, count * 1000.0 / (clock() - ctime));
}
char** read_lines(const char* filename)
@@ -171,9 +176,8 @@ char** read_lines(const char* filename)
buf[strcspn(buf, "\n")] = 0;
lines[i] = strdup(buf);
-
double elapsed = (clock() - lastPrint) * 1000.0 / CLOCKS_PER_SEC;
- if (elapsed > 50)
+ if (print_progress && elapsed > 50)
{
fprintf(stderr, "\rRead %d lines from %s in %.3f ms\r", i + 1, filename, (clock() - ctime) * 1000.0 / CLOCKS_PER_SEC);
lastPrint = clock();
@@ -199,6 +203,11 @@ char** grow_string_array(char** array, size_t newSize)
return newArray;
}
+void ban_ip(char* ip)
+{
+
+}
+
void process_line(char* line, void* cbData)
{
MatchRules* rules = cbData;
@@ -269,6 +278,8 @@ void process_line(char* line, void* cbData)
if (print_bans)
printf("[Bans=%6lu] %15s matched pattern %4d (\"%s\"): %s\n", rules->bannedIpCount, ip, i, rules->patterns[i], line);
+ if (print_only) return;
+
__pid_t pid = fork();
if (pid == 0)
{
@@ -300,11 +311,12 @@ void process_line(char* line, void* cbData)
int main(int argc, char* argv[])
{
- printf("Hello, World!\n");
+ fprintf(stderr, "Hello, World!\n");
print_allowed_ip = getenv("D_PRINT_ALLOWED_IP") != NULL && strcmp(getenv("D_PRINT_ALLOWED_IP"), "1") == 0;
print_allowed_pattern = getenv("D_PRINT_ALLOWED_PATTERN") != NULL && strcmp(getenv("D_PRINT_ALLOWED_PATTERN"), "1") == 0;
print_already_banned_ip = getenv("D_PRINT_BANNED_IP") != NULL && strcmp(getenv("D_PRINT_BANNED_IP"), "1") == 0;
+ print_progress = isatty(STDERR_FILENO);
for (int i = 1; i < argc; i++)
{
@@ -326,6 +338,12 @@ int main(int argc, char* argv[])
print_bans = strstr(argv[i], "=false") == NULL;
else if (strstr(argv[i], "--print-indeterminate") != NULL)
print_indeterminate = strstr(argv[i], "=false") == NULL;
+ else if (strstr(argv[i], "--print-only") != NULL)
+ print_only = strstr(argv[i], "=false") == NULL;
+ else if (strstr(argv[i], "--print-progress") != NULL)
+ print_progress = strstr(argv[i], "=false") == NULL;
+ else if (strstr(argv[i], "--dump-patterns") != NULL)
+ dump_patterns = true;
else if (strcmp(argv[i], "--debug") == 0)
{
print_allowed_ip = true;
@@ -334,6 +352,25 @@ int main(int argc, char* argv[])
print_indeterminate = true;
print_bans = true;
}
+ else if (strstr(argv[i], "--help"))
+ {
+ printf("Usage: %s [options]\n", argv[0]);
+ printf("Options:\n");
+ printf(" --allowed-ips <path> Path to the allowed IPs file (default: %s)\n", ALLOWED_IPS_FILE);
+ printf(" --allowed-patterns <path> Path to the allowed patterns file (default: %s)\n", ALLOWED_PATTERNS_FILE);
+ printf(" --patterns <path> Path to the patterns file (default: %s)\n", PATTERNS_FILE);
+ printf(" --access-log <path> Path to the access log file (default: %s)\n", ACCESS_LOG_FILE);
+ printf(" --print-allowed-ip[=false] Print allowed IPs (default: %s)\n", print_allowed_ip ? "true" : "false");
+ printf(" --print-allowed-pattern[=false] Print allowed patterns (default: %s)\n", print_allowed_pattern ? "true" : "false");
+ printf(" --print-already-banned-ip[=false] Print already banned IPs (default: %s)\n", print_already_banned_ip ? "true" : "false");
+ printf(" --print-bans[=false] Print bans (default: %s)\n", print_bans ? "true" : "false");
+ printf(" --print-indeterminate[=false] Print indeterminate lines (default: %s)\n", print_indeterminate ? "true" : "false");
+ printf(" --print-only[=false] Print only the lines that match the rules (default: %s)\n", print_only ? "true" : "false");
+ printf(" --print-progress[=false] Print progress while reading file (default: %s)\n", print_progress ? "true" : "false");
+ printf(" --debug Enable debug mode (prints all information)\n");
+ printf(" --dump-patterns Dump pattern table and exit\n");
+ return 0;
+ }
else
{
fprintf(stderr, "Unknown argument: %s\n", argv[i]);
@@ -366,8 +403,28 @@ int main(int argc, char* argv[])
rules.bannedIps = xmalloc(sizeof(char**) * rules.bannedIpSize);
//rules.bannedIps[0] = ;
+ if (dump_patterns)
+ {
+ fprintf(stderr, "Dumping patterns:\n");
+ for (int i = 0; rules.patterns[i] != NULL; i++)
+ {
+ printf("[PATTERN %4d] %s\n", i, rules.patterns[i]);
+ }
+ for (int i = 0; rules.allowed_patterns[i] != NULL; i++)
+ {
+ printf("[ALLOWED %4d] %s\n", i, rules.allowed_patterns[i]);
+ }
+ for (int i = 0; rules.allowedIps[i] != NULL; i++)
+ {
+ printf("[ALL. IP %4d] %s\n", i, rules.allowedIps[i]);
+ }
+ return 0;
+ }
+
read_lines_cb(access_log_path, process_line, &rules);
+ fprintf(stderr, "Banned %lu IPs:\n", rules.bannedIpCount);
+
for (int i = 0; rules.bannedIps[i] != NULL; i++)
{
free(rules.bannedIps[i]);
diff --git a/main.sh b/main.sh
index 02565de..61853c6 100755
--- a/main.sh
+++ b/main.sh
@@ -1,6 +1,6 @@
#! /usr/bin/env sh
./build_patterns.sh
-gcc -O0 -ggdb -std=c23 main.c || exit 1
+gcc -O0 -ggdb -std=c23 -pie -fPIE main.c || exit 1
#sudo valgrind --leak-check=full -s ./a.out
rsync -raPz *.c *.txt a.out rory.gay:
diff --git a/meson.build b/meson.build
new file mode 100644
index 0000000..241a1cf
--- /dev/null
+++ b/meson.build
@@ -0,0 +1,7 @@
+project('BlockScrapersC', 'c',
+ version : '1.0.0',
+ default_options : ['warning_level=3', 'c_std=c2x'])
+
+BlockScrapersC = executable('BlockScrapersC', 'main.c', install : true)
+
+test('test', BlockScrapersC)
\ No newline at end of file
diff --git a/patterns.txt b/patterns.txt
deleted file mode 100644
index 97ff2bc..0000000
--- a/patterns.txt
+++ /dev/null
@@ -1,878 +0,0 @@
-\x00
-\x01
-\x02
-\x03
-\x04
-\x05
-\x06
-\x07
-\x08
-\x09
-\x0A
-\x0B
-\x0C
-\x0D
-\x0E
-\x0F
-\x10
-\x11
-\x12
-\x13
-\x14
-\x15
-\x16
-\x17
-\x18
-\x19
-\x1A
-\x1B
-\x1C
-\x1D
-\x1E
-\x1F
-\x20
-\x21
-\x22
-\x23
-\x24
-\x25
-\x26
-\x27
-\x28
-\x29
-\x2A
-\x2B
-\x2C
-\x2D
-\x2E
-\x2F
-\x30
-\x31
-\x32
-\x33
-\x34
-\x35
-\x36
-\x37
-\x38
-\x39
-\x3A
-\x3B
-\x3C
-\x3D
-\x3E
-\x3F
-\x40
-\x41
-\x42
-\x43
-\x44
-\x45
-\x46
-\x47
-\x48
-\x49
-\x4A
-\x4B
-\x4C
-\x4D
-\x4E
-\x4F
-\x50
-\x51
-\x52
-\x53
-\x54
-\x55
-\x56
-\x57
-\x58
-\x59
-\x5A
-\x5B
-\x5C
-\x5D
-\x5E
-\x5F
-\x60
-\x61
-\x62
-\x63
-\x64
-\x65
-\x66
-\x67
-\x68
-\x69
-\x6A
-\x6B
-\x6C
-\x6D
-\x6E
-\x6F
-\x70
-\x71
-\x72
-\x73
-\x74
-\x75
-\x76
-\x77
-\x78
-\x79
-\x7A
-\x7B
-\x7C
-\x7D
-\x7E
-\x7F
-\x80
-\x81
-\x82
-\x83
-\x84
-\x85
-\x86
-\x87
-\x88
-\x89
-\x8A
-\x8B
-\x8C
-\x8D
-\x8E
-\x8F
-\x90
-\x91
-\x92
-\x93
-\x94
-\x95
-\x96
-\x97
-\x98
-\x99
-\x9A
-\x9B
-\x9C
-\x9D
-\x9E
-\x9F
-\xA0
-\xA1
-\xA2
-\xA3
-\xA4
-\xA5
-\xA6
-\xA7
-\xA8
-\xA9
-\xAA
-\xAB
-\xAC
-\xAD
-\xAE
-\xAF
-\xB0
-\xB1
-\xB2
-\xB3
-\xB4
-\xB5
-\xB6
-\xB7
-\xB8
-\xB9
-\xBA
-\xBB
-\xBC
-\xBD
-\xBE
-\xBF
-\xC0
-\xC1
-\xC2
-\xC3
-\xC4
-\xC5
-\xC6
-\xC7
-\xC8
-\xC9
-\xCA
-\xCB
-\xCC
-\xCD
-\xCE
-\xCF
-\xD0
-\xD1
-\xD2
-\xD3
-\xD4
-\xD5
-\xD6
-\xD7
-\xD8
-\xD9
-\xDA
-\xDB
-\xDC
-\xDD
-\xDE
-\xDF
-\xE0
-\xE1
-\xE2
-\xE3
-\xE4
-\xE5
-\xE6
-\xE7
-\xE8
-\xE9
-\xEA
-\xEB
-\xEC
-\xED
-\xEE
-\xEF
-\xF0
-\xF1
-\xF2
-\xF3
-\xF4
-\xF5
-\xF6
-\xF7
-\xF8
-\xF9
-\xFA
-\xFB
-\xFC
-\xFD
-\xFE
-\xFF
-;/actuator
-;/env
-;/internal
-;/META-INF
-/phpunit
-eval-stdin.php
-phpinfo
-/.git/
-.env
-.htaccess
-web.config
-/.svn/
-/.AWS_/credentials
-/.DS_Store
-/.__info.php
-/.aws/
-/.circleci/configs/development.yml
-/.config/sftp.json
-/.dockerignore
-/.npmrc
-/.sendgrid
-/.travis.yml
-/.vscode/sftp.json
- /%61%63%74%75%61%74%6f%72/%65%6e%76
- /%61%70%69/%61%63%74%75%61%74%6f%72/%65%6e%37%36
- /%61%70%69/%61%63%74%75%61%74%6f%72/%65%6e%76
- /%61%70%69/%65%6e%37%36
- /%61%70%69/%65%6e%76
- /%61%70%69/%69%6e%74%65%72%6e%61%6c/%61%63%74%75%61%74%6f%72/%65%6e%76
- /%65%6e%76
- /%67%61%74%65%77%61%79/%61%63%74%75%61%74%6f%72/%65%6e%76
- /%67%61%74%65%77%61%79/%65%6e%76
- /%6d%61%6e%61%67%65%6d%65%6e%74/%61%63%74%75%61%74%6f%72/%65%6e%76
- /%6d%61%6e%61%67%65%6d%65%6e%74/%65%6e%76
- /%6d%61%6e%61%67%65/%61%63%74%75%61%74%6f%72/%65%6e%76
- /%6d%61%6e%61%67%65/%65%6e%76
- /+CSCOE+/logon.html
- /.wp-config.php.swp
- //admin/login.asp
- //installer.php
- //webpages/login.html
- /0-info.php
- /00_server_info.php
- /01-info.php
- /0_info.php
- /0info.php
- /1.php
- /1_1_PhpInfo.php
- /2018/wp-includes/wlwmanifest.xml
- /2019/wp-includes/wlwmanifest.xml
- /3/3/3/3/3/3/3/3/3/3/3/3/3/3/3/3
- /5info.php
- /@vite/env
- /Account/Login
- /AwsConfig.json
- /Awsconfig.json
- /CFIDE/componentutils/
- /CHANGELOG.txt
- /CSS/Miniweb.css
- /Dockerfile
- /HNAP1
- /IPCamDesc.xml
- /NuLM
- /OA_HTML/AppsLocalLogin.jsp
- /PHPInfo.php
- /PSIA/index
- /PhpInfo.php
- /Portal/Portal.mwsl
- /Portal0000.htm
- /Public/home/js/check.js
- /RDWeb/
- /README.md
- /ReportServer
- /Telerik.Web.UI.WebResource.axd
- /WebApp/js/UI_String.js
- /WebInterface/
- /__Additional
- /_all_dbs
- /_phpinf.php
- /_wpeprivate/config.json
- /aaa9
- /aab8
- /aab9
- /ab2g
- /ab2h
- /about
- /actuator/env
- /actuator/gateway/routes
- /actuator/health
- /admin.asp
- /admin.cfm
- /admin.cgi
- /admin.html
- /admin.jsa
- /admin.jsp
- /admin.php
- /admin.pl
- /admin/
- /administrator/index.php
- /administrator/manifests/files/joomla.xml
- /ads.txt
- /alive.php
- /allversions
- /api-docs
- /api/actuator/env
- /api/config.js
- /api/config/config.yml
- /api/credentials
- /api/env
- /api/objects/codes.php.save
- /api/proxy
- /api/server/version
- /api/session/properties
- /api/sonicos/auth
- /api/sonicos/is-sslvpn-enabled
- /api/sonicos/tfa
- /api/v1.0/environment
- /api/v1/check-version
- /api/v1/label/__name__/values
- /api/v1/metadata
- /api/v1/notifications/live
- /api/v1/proxy
- /api/v1/query
- /api/v1/status/buildinfo
- /api/v1/status/runtimeinfo
- /api/v1/ws/server
- /api/v2/about
- /api/v2/proxy
- /api/v3/meta
- /api/version
- /api/vip/i18n/api/v2/translation/products/vRNIUI/versions/1
- /api_keys/sendgrid_keys.json
- /apis/config/config.js
- /apis/controllers/users.js
- /app.js
- /app.py
- /app/config/parameters.yml
- /app/etc/env.php
- /app/etc/local.xml
- /application.properties
- /application.yml
- /apps/zxtm/login.cgi
- /aspera/faspex/
- /assets/env.js
- /assets/index-BPbBbNOr.css
- /assets/index-DuE_NgAI.js
- /aura
- /auth.html
- /auth.json
- /auth1.html
- /autodiscover/autodiscover.json
- /aws-secret.yaml
- /aws.yml
- /aws/credentials
- /backend/config/default.yml
- /backend/config/development.yml
- /backend/config/settings.yml
- /backup
- /base.cfm
- /base.jsa
- /base.shtml
- /baseDstu2/metadata
- /baseDstu3/metadata
- /baseR2/metadata
- /baseR3/metadata
- /baseR4/metadata
- /baseR5/metadata
- /bitrix/php_interface/dbconn.php
- /blog
- /boaform/admin/formLogin
- /bootstrap/cache/config.php
- /build.gradle
- /c/login
- /cdn-cgi/trace/cdn-cgi/trace
- /centreon/api/latest/platform/versions
- /cf_scripts/scripts/ajax/ckeditor/ckeditor.js
- /cgi-bin/
- /cloud/Scraper.js
- /cluster/list.query
- /cms/wp-includes/wlwmanifest.xml
- /composer.json
- /computeMetadata/v1
- /config
- /confluence/rest/applinks/1.0/manifest
- /contact/
- /containers/json
- /controller/admin/post.js
- /controller/api/post.js
- /controllers/settings.js
- /core/install.php
- /cslu/v1/core/conf
- /css/elfinder.min.css
- /css/eonweb.css
- /css/images/PTZOptics_powerby.png
- /dana-cached/hc/HostCheckerInstaller.osx
- /dana-na/nc/nc_gina_ver.txt
- /database.json
- /db.ini
- /db_backup.sql
- /debug.php
- /debug/default/view
- /default.html
- /default.jhtml
- /default.jsa
- /developmentserver/metadatauploader
- /dniapi/userInfos
- /dns-query
- /doc/index.html
- /docker-compose.
- /docs/cplugError.html/
- /druid/index.html
- /ecp/Current/exporttool/microsoft.exchange.ediscovery.exporttool.application
- /en%76;
- /env
- /epa/scripts/win/nsepa_setup.exe
- /etc/gitconfig
- /evox/about
- /ext-js/app/common/zld_product_spec.js
- /fhir-server/api/v4/metadata
- /fhir/metadata
- /fog/management/index.php
- /form.html
- /forms/doLogin
- /ftptest.cgi
- /gatsby-config.js
- /geoip/
- /geoserver
- /getcpuutil.php-bakworking
- /git/.config
- /global-protect/login.esp
- /health
- /hello.world
- /helm/values.yaml
- /helpdesk/WebObjects/Helpdesk.woa
- /helper.js
- /helper/EmailHelper.js
- /helpers/utility.js
- /home.aspx
- /home.cfm
- /home.html
- /hudson
- /human.aspx
- /i.php
- /identity
- /index.aspx
- /index.cfm
- /index.cgi
- /index.jsp
- /index.php
- /index.pl
- /index.shtml
- /indice.cfm
- /indice.cgi
- /indice.html
- /info
- /inicio.php
- /inicio.pl
- /instance/sendgrid_keys.py
- /internal/api
- /internal/proxy
- /internal_forms_authentication
- /jasperserver-pro/login.html
- /jasperserver/login.html
- /jasperserverTest/login.html
- /javascript/validation/OEM.js
- /joomla/configuration.php-dist
- /js/NewWindow_2_all.js
- /js/app.js
- /js/config.js
- /js/main.js
- /karma.conf.json
- /keys/sendgrid_keys.json
- /kylin/
- /language/en-GB/en-GB.xml
- /lara/info.php
- /laravel/info.php
- /latest/meta-data
- /latest/user-data
- /lms/db
- /local.inc.php
- /local.ini
- /local.json
- /local.php
- /local.xml
- /local_settings.py
- /localstart.jsa
- /login.action
- /login.asp
- /login.do
- /login.htm
- /login.jsp
- /login.php
- /login/login.html
- /logon/LogonPoint/index.html
- /logs/error.php
- /magento_version
- /main.asp
- /main.cfm
- /main.js
- /main.yml
- /manage/account/login
- /manager/html
- /media/wp-includes/wlwmanifest.xml
- /menu.aspx
- /menu.cfm
- /menu.php
- /metadata
- /modules/contrib/sendgrid_mail/
- /my_env/chakaash.py
- /my_env/newsletter.py
- /my_env/palash.py
- /mytest/astech_robot.js
- /new
- /nmaplowercheck1745842941
- /nova-api/styles
- /o8kJ
- /odinhttpcall1746060903
- /odinhttpcall1746102992
- /officescan/console/cgi/cgiChkMasterPwd.exe
- /officescan/console/html/localization.js
- /old
- /onvif/device_service
- /openapi.json
- /openapi/v2
- /opt/aws/
- /owa/
- /owncloud/status.php
- /p/login/
- /package.json
- /pandora_console/
- /parameters.yaml
- /parameters.yml
- /partner/config/config.js
- /partymgr/control/main
- /password.php
- /php.ini
- /php.php
- /php/ztp_gate.php/.js.map
- /php_info
- /phpmyadmin/index.php
- /phpmyadmin4.8.5/index.php
- /pi.php
- /pmd/index.php
- /pom.xml
- /pools
- /portal
- /private/
- /proxy
- /public/index.php
- /public/js/main.js
- /query
- /r-seenet/index.php
- /r2/metadata
- /r3/metadata
- /r4/metadata
- /r5/metadata
- /readme.txt
- /register/
- /remote/fgt_lang
- /remote/login
- /resolve
- /rest/applinks/1.0/manifest
- /root/info.php
- /root/infophp
- /s/aura
- /s/fact
- /s/sfsites/aura
- /s3.js
- /scripts/WPnBr.dll
- /scripts/nodemailer.js
- /sdk
- /secret
- /secure-config.json
- /sendgrid
- /server-info
- /server-status
- /server.js
- /server.php
- /server/config/database.js
- /server/s3.js
- /server_info.php
- /service/email_service.py
- /settings.bak
- /settings.cfg
- /settings.inc.php
- /settings.ini
- /settings.json
- /settings.php
- /settings.py
- /settings.xml
- /settings.yaml
- /settings/sendgrid_config
- /settings/sendgrid_keys
- /setup.cgi
- /sfsites/aura
- /sftp-config.json
- /sftp.json
- /shared/config/config.js
- /shop/wp-includes/wlwmanifest.xml
- /showLogin.cc
- /site/wp-includes/wlwmanifest.xml
- /sitecore/shell/sitecore.version.xml
- /sitemap.xml
- /sites/all/modules/
- /sites/all/themes/
- /sites/default/settings.php
- /sito/wp-includes/wlwmanifest.xml
- /sms.py
- /solr/
- /sonicui/7/login/
- /src/config.js
- /sslvpnLogin.html
- /sslvpn_logon.shtml
- /static/admin/javascript/hetong.js
- /static/historypage.js
- /stats
- /status
- /storage/framework/cache/
- /storage/framework/sessions/
- /storage/framework/views/
- /storage/logs/laravel.log
- /storage/sendgrid.json
- /sugar_version.json
- /swagger-ui.html
- /swagger.js
- /systembc/password.php
- /t4
- /telescope/requests
- /temp
- /teorema505
- /test
- /tmp.php
- /tos/index.php
- /typo3conf/localconf.php
- /ucT0
- /upl.php
- /user
- /v1
- /v2/_catalog
- /var/aws/
- /var/lib/aws/
- /var/log/
- /var/logs/
- /versa/login
- /version
- /vpn/index.html
- /vpnsvc/connect.cgi
- /wRGL
- /web/
- /webportal.cgi
- /website/wp-includes/wlwmanifest.xml
- /webui
- /wordpress
- /workplace/home.action
- /wp
- /wsman
- /xml/info.xml
- /xmldata
- /xmlrpc.php
- /yarn.lock
- /zabbix/favicon.ico
- <NULL>
- "CONNECT
-# OpenAI
-+https://openai.com/gptbot
-OAI-SearchBot/
-# Claude
-+claudebot@anthropic.com
-ClaudeBot/1.0
-AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com)
-Mozilla/5.0 zgrab/
-Mozilla/5.0; Keydrop.io/
-FH Muenster/Security-Scanner/fh-muenster.de
-"'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/genomecrawler)'"
-l9explore/1.2.2
-"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"
-"Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)"
- abuse.xmco.fr"
-+https://developer.amazon.com/support/amazonbot
-http://mj12bot.com/
-SeznamBot/
-SemanticScholarBot
-NetcraftSurveyAgent
-ALittle Client
-scaninfo@paloaltonetworks.com
-Scrapy/
-CensysInspect/
-ModatScanner/
-AppleBot
-AhrefsBot
- Chrome/10.
- Chrome/11.
- Chrome/12.
- Chrome/13.
- Chrome/14.
- Chrome/15.
- Chrome/16.
- Chrome/17.
- Chrome/18.
- Chrome/19.
- Chrome/20.
- Chrome/21.
- Chrome/22.
- Chrome/23.
- Chrome/24.
- Chrome/25.
- Chrome/26.
- Chrome/27.
- Chrome/28.
- Chrome/29.
- Chrome/3
- Chrome/4
- Chrome/5
- Chrome/6
- Chrome/7
- Chrome/8
- Chrome/9
- CriOS/10.
- CriOS/11.
- CriOS/12.
- CriOS/13.
- CriOS/14.
- CriOS/15.
- CriOS/16.
- CriOS/17.
- CriOS/18.
- CriOS/19.
- CriOS/20.
- CriOS/21.
- CriOS/22.
- CriOS/23.
- CriOS/24.
- CriOS/25.
- CriOS/26.
- CriOS/27.
- CriOS/28.
- CriOS/29.
- CriOS/3
- CriOS/4
- CriOS/5
- CriOS/6
- CriOS/7
- CriOS/8
- CriOS/9
- Firefox/10.
- Firefox/11.
- Firefox/12.
- Firefox/13.
- Firefox/14.
- Firefox/15.
- Firefox/16.
- Firefox/17.
- Firefox/18.
- Firefox/19.
- Firefox/20.
- Firefox/21.
- Firefox/22.
- Firefox/23.
- Firefox/24.
- Firefox/25.
- Firefox/26.
- Firefox/27.
- Firefox/28.
- Firefox/29.
- Firefox/3
- Firefox/4
- Firefox/5
- Firefox/6
- Firefox/7
- Firefox/8
- Firefox/9
- FxiOS/10.
- FxiOS/11.
- FxiOS/12.
- FxiOS/13.
- FxiOS/14.
- FxiOS/15.
- FxiOS/16.
- FxiOS/17.
- FxiOS/18.
- FxiOS/19.
- FxiOS/20.
- FxiOS/21.
- FxiOS/22.
- FxiOS/23.
- FxiOS/24.
- FxiOS/25.
- FxiOS/26.
- FxiOS/27.
- FxiOS/28.
- FxiOS/29.
- FxiOS/3
- FxiOS/4
- FxiOS/5
- FxiOS/6
- FxiOS/7
- FxiOS/8
- FxiOS/9
- Presto/
- Trident/
-PPC Mac OS X
-Win 9x
-Windows CE
-Windows 95
-Windows 98
-Windows NT 4
-Windows NT 5
-Windows NT 6
-Windows NT 10.0; Trident
-CPU iPhone OS 3
-CPU iPhone OS 4
-CPU iPhone OS 5
-CPU iPhone OS 6
-CPU iPhone OS 7
-CPU iPhone OS 8
-CPU iPhone OS 9
-Android 2.
-Android 3.
-Android 4.
-Android 5.
-Android 6.
-MSIE 6.0
diff --git a/template/ap_emma.txt b/template/ap_emma.txt
index a1ff752..2412087 100644
--- a/template/ap_emma.txt
+++ b/template/ap_emma.txt
@@ -1,2 +1,5 @@
GET /server.git/
-GET /matrix/thirdparty/nheko.git/
\ No newline at end of file
+GET /matrix/thirdparty/nheko.git/
+# git:
+"-" "git/
+"Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
\ No newline at end of file
diff --git a/template/ap_matrix.txt b/template/ap_matrix.txt
index b422bd3..c1349d9 100644
--- a/template/ap_matrix.txt
+++ b/template/ap_matrix.txt
@@ -2,4 +2,4 @@
/_matrix/client/
/_matrix/federation/
/_matrix/key
-
+"Synapse (bot; +https://github.com/matrix-org/synapse)"
\ No newline at end of file
diff --git a/template/ap_owncast.txt b/template/ap_owncast.txt
new file mode 100644
index 0000000..8107640
--- /dev/null
+++ b/template/ap_owncast.txt
@@ -0,0 +1,8 @@
+# Get stream metadata
+"GET /hls/stream.m3u8
+"GET /hls/0/stream.m3u8
+# mpv
+"-" "libmpv"
+# initiated by... is this too loose?
+ "https://stream.rory.gay/sw.js"
+ "https://stream.rory.gay/"
\ No newline at end of file
diff --git a/template/p_paths.txt b/template/p_paths.txt
index 9508814..59cf8d1 100644
--- a/template/p_paths.txt
+++ b/template/p_paths.txt
@@ -437,3 +437,4 @@
/yarn.lock
/zabbix/favicon.ico
<NULL>
+ //recordings/theme/main.css
\ No newline at end of file
diff --git a/template/p_separate.txt b/template/p_separate.txt
index f7b9062..a787b04 100644
--- a/template/p_separate.txt
+++ b/template/p_separate.txt
@@ -1 +1,4 @@
"CONNECT
+ "SSTP_DUPLEX_POST
+# Why are you doing direct connections?
+{host="51.210.113.110"
\ No newline at end of file
diff --git a/template/p_user_agent_bot.txt b/template/p_user_agent_bot.txt
index 08ab337..543adbb 100644
--- a/template/p_user_agent_bot.txt
+++ b/template/p_user_agent_bot.txt
@@ -1,20 +1,34 @@
# OpenAI
+https://openai.com/gptbot
++https://openai.com/bot
OAI-SearchBot/
# Claude
+claudebot@anthropic.com
ClaudeBot/1.0
-
-AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com)
+# Facebook
+facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
+facebookexternalhit/1.1
+facebookcatalog/1.0
+meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)
+meta-externalagent/1.1
+meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)
+meta-externalfetcher/1.1
+# Other (by linked help domain)
++http://yandex.com
++https://developer.amazon.com
++https://search.brave.com
++https://opensiteexplorer.org
+# Other (mozilla/5.0)
Mozilla/5.0 zgrab/
Mozilla/5.0; Keydrop.io/
-FH Muenster/Security-Scanner/fh-muenster.de
"'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/genomecrawler)'"
-l9explore/1.2.2
"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"
"Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)"
+# "Security scanners"
+AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com)
+FH Muenster/Security-Scanner/fh-muenster.de
+l9explore/1.2.2
abuse.xmco.fr"
-+https://developer.amazon.com/support/amazonbot
http://mj12bot.com/
SeznamBot/
SemanticScholarBot
@@ -26,3 +40,5 @@ CensysInspect/
ModatScanner/
AppleBot
AhrefsBot
+# Unknown and nondescript
+ "link_checker/0.1.0"
diff --git a/template/p_user_agent_legacy_browser.txt b/template/p_user_agent_legacy_browser.txt
index a318b0c..a882dc0 100644
--- a/template/p_user_agent_legacy_browser.txt
+++ b/template/p_user_agent_legacy_browser.txt
@@ -1,6 +1,6 @@
- Chrome/10.
- Chrome/11.
- Chrome/12.
+ Chrome/10
+ Chrome/11
+ Chrome/12
Chrome/13.
Chrome/14.
Chrome/15.
@@ -8,16 +8,7 @@
Chrome/17.
Chrome/18.
Chrome/19.
- Chrome/20.
- Chrome/21.
- Chrome/22.
- Chrome/23.
- Chrome/24.
- Chrome/25.
- Chrome/26.
- Chrome/27.
- Chrome/28.
- Chrome/29.
+ Chrome/2
Chrome/3
Chrome/4
Chrome/5
@@ -52,9 +43,9 @@
CriOS/7
CriOS/8
CriOS/9
- Firefox/10.
- Firefox/11.
- Firefox/12.
+ Firefox/10
+ Firefox/11
+ Firefox/12
Firefox/13.
Firefox/14.
Firefox/15.
@@ -62,16 +53,7 @@
Firefox/17.
Firefox/18.
Firefox/19.
- Firefox/20.
- Firefox/21.
- Firefox/22.
- Firefox/23.
- Firefox/24.
- Firefox/25.
- Firefox/26.
- Firefox/27.
- Firefox/28.
- Firefox/29.
+ Firefox/2
Firefox/3
Firefox/4
Firefox/5
@@ -108,3 +90,9 @@
FxiOS/9
Presto/
Trident/
+ Opera 1.
+ Opera 2.
+ Opera 3.
+ Opera 4.
+ Opera 5.
+ Opera 6.
\ No newline at end of file
diff --git a/template/p_user_agent_legacy_os.txt b/template/p_user_agent_legacy_os.txt
index ea58417..30966ba 100644
--- a/template/p_user_agent_legacy_os.txt
+++ b/template/p_user_agent_legacy_os.txt
@@ -1,8 +1,22 @@
PPC Mac OS X
+Mac OS X 10_1_
+Mac OS X 10_2_
+Mac OS X 10_3
+Mac OS X 10_4
+Mac OS X 10_5
+Mac OS X 10_6
+Mac OS X 10_7
+Mac OS X 10_8
+Mac OS X 10_9
+Mac OS X 10_10
+Mac OS X 10_11
+Mac OS X 10_12
+Mac OS X 10_13
Win 9x
Windows CE
Windows 95
Windows 98
+Windows XP
Windows NT 4
Windows NT 5
Windows NT 6
@@ -14,9 +28,34 @@ CPU iPhone OS 6
CPU iPhone OS 7
CPU iPhone OS 8
CPU iPhone OS 9
+CPU iPhone OS 10
+CPU iPhone OS 11
+CPU iPhone OS 12
+CPU iPhone OS 13
+CPU iPhone OS 14
+CPU iPhone OS 15
+CPU iPhone OS 16
+CPU iPad OS 3
+CPU iPad OS 4
+CPU iPad OS 5
+CPU iPad OS 6
+CPU iPad OS 7
+CPU iPad OS 8
+CPU iPad OS 9
+CPU iPad OS 10
+CPU iPad OS 11
+CPU iPad OS 12
+CPU iPad OS 13
+CPU iPad OS 14
+CPU iPad OS 15
+CPU iPad OS 16
Android 2.
Android 3.
Android 4.
Android 5.
Android 6.
+Android 7.
+Android 8.
+Android 9.
+Android 10.
MSIE 6.0
diff --git a/template/p_user_agent_likely_fake.txt b/template/p_user_agent_likely_fake.txt
new file mode 100644
index 0000000..9e762fe
--- /dev/null
+++ b/template/p_user_agent_likely_fake.txt
@@ -0,0 +1,3 @@
+.0.0.0
+(Windows; U; Windows NT 10.0) AppleWebKit/
+(Windows; U; Windows NT 11.0) AppleWebKit/
\ No newline at end of file
diff --git a/template/p_user_agent_unknown.txt b/template/p_user_agent_unknown.txt
new file mode 100644
index 0000000..f937ab3
--- /dev/null
+++ b/template/p_user_agent_unknown.txt
@@ -0,0 +1 @@
+"Linux Gnu (cow)"
\ No newline at end of file
|
