summary refs log tree commit diff
diff options
context:
space:
mode:
authorRory& <root@rory.gay>2025-05-13 08:09:14 +0200
committerRory& <root@rory.gay>2025-05-13 08:09:14 +0200
commit933a89c8a18b9bb5376bee32313f061d624f66da (patch)
treeffc74679864b41296f74f0948929907a7cdb22f8
parentInitial commit (diff)
downloadBlockScrapersC-933a89c8a18b9bb5376bee32313f061d624f66da.tar.xz
Pattern update, add cli flags
-rw-r--r--allowed_ips.txt (renamed from ai_emma.txt)0
-rw-r--r--allowed_patterns.txt6
-rwxr-xr-xbuild_patterns.sh6
-rw-r--r--main.c122
-rw-r--r--p_user_agent_legacy_browser.txt3
-rw-r--r--p_user_agent_legacy_win.txt0
-rw-r--r--patterns.txt878
-rw-r--r--template/ai_emma.txt4
-rw-r--r--template/ap_emma.txt (renamed from ap_emma.txt)0
-rw-r--r--template/ap_matrix.txt (renamed from ap_matrix.txt)2
-rw-r--r--template/p_escaped_hex.txt256
-rw-r--r--template/p_path_contains.txt (renamed from p_path_contains.txt)0
-rw-r--r--template/p_paths.txt (renamed from p_paths.txt)0
-rw-r--r--template/p_separate.txt (renamed from p_separate.txt)0
-rw-r--r--template/p_user_agent_bot.txt (renamed from p_user_agents.txt)19
-rw-r--r--template/p_user_agent_legacy_browser.txt110
-rw-r--r--template/p_user_agent_legacy_os.txt22
17 files changed, 1402 insertions, 26 deletions
diff --git a/ai_emma.txt b/allowed_ips.txt

index 849702f..849702f 100644 --- a/ai_emma.txt +++ b/allowed_ips.txt
diff --git a/allowed_patterns.txt b/allowed_patterns.txt new file mode 100644
index 0000000..fe74fac --- /dev/null +++ b/allowed_patterns.txt
@@ -0,0 +1,6 @@ +GET /server.git/ +GET /matrix/thirdparty/nheko.git/ + /.well-known/matrix/ + /_matrix/client/ + /_matrix/federation/ + /_matrix/key diff --git a/build_patterns.sh b/build_patterns.sh
index 5b41ac3..92a4782 100755 --- a/build_patterns.sh +++ b/build_patterns.sh
@@ -1,5 +1,5 @@ #! /usr/bin/env bash -cat ai_* > allowed_ips.txt -cat ap_* > allowed_patterns.txt -cat p_* > patterns.txt \ No newline at end of file +awk NF template/ai_* > allowed_ips.txt +awk NF template/ap_* > allowed_patterns.txt +awk NF template/p_* > patterns.txt diff --git a/main.c b/main.c
index e59aef8..18e9f80 100644 --- a/main.c +++ b/main.c
@@ -5,12 +5,13 @@ #include <time.h> #include <unistd.h> #include <pthread.h> +#include <wait.h> #define ALLOWED_PATTERNS_FILE "allowed_patterns.txt" #define ALLOWED_IPS_FILE "allowed_ips.txt" #define PATTERNS_FILE "patterns.txt" -#define ACCESS_LOG_FILE "access.log.1" -// #define ACCESS_LOG_FILE "/var/log/nginx/access.log" +// #define ACCESS_LOG_FILE "access.log.1" +#define ACCESS_LOG_FILE "/var/log/nginx/access.log" typedef struct { @@ -23,11 +24,16 @@ typedef struct size_t bannedIpSize; } MatchRules; - bool print_allowed_ip = false; bool print_allowed_pattern = false; -bool print_banned_ip = false; -// bool print_allowed_ip = strcmp(getenv("PRINT_"), "1") == 0; +bool print_already_banned_ip = false; +bool print_indeterminate = false; +bool print_bans = true; + +char* allowed_patterns_path = ALLOWED_PATTERNS_FILE; +char* allowed_ips_path = ALLOWED_IPS_FILE; +char* patterns_path = PATTERNS_FILE; +char* access_log_path = ACCESS_LOG_FILE; void* xmalloc(size_t size) { @@ -40,6 +46,26 @@ void* xmalloc(size_t size) return ptr; } +void signal_handler(int signal_number) +{ + int wait_status; + pid_t return_pid = wait(&wait_status); + if (return_pid == -1) + { + perror("wait()"); + } + if (WIFEXITED(wait_status)) + { + printf("job [%d] | pid: %d | exit status: %d\n", signal_number, return_pid, WEXITSTATUS(wait_status)); + } + else + { + printf("exit abnormally\n"); + } + + fprintf(stderr, "the signal %d was received\n", signal_number); +} + void read_lines_cb(const char* filename, void cb(char*, void*), void* cbData) { if (cb == nullptr) @@ -199,19 +225,18 @@ void process_line(char* line, void* cbData) if (strstr(ip, rules->allowedIps[j]) == ip) { if (print_allowed_ip) - printf("Allowed IP: \"%s\" (~%s): %s\r", ip, rules->allowedIps[j], line); + printf("Allowed IP: \"%s\" (~%s): %s\n", ip, rules->allowedIps[j], line); free(ip); return; } } - // printf("Checking IP %s against %d banned IPs\n", ip, rules->bannedIpCount); for (size_t j = 0; j < rules->bannedIpCount; j++) { if (strstr(ip, rules->bannedIps[j]) == ip) { - if (print_banned_ip) - printf("Banned IP: \"%s\": %s\r", rules->bannedIps[j], line); + if (print_already_banned_ip) + printf("Banned IP: \"%s\": %s\n", rules->bannedIps[j], line); free(ip); return; } @@ -241,10 +266,9 @@ void process_line(char* line, void* cbData) rules->bannedIps[rules->bannedIpCount][strcspn(rules->bannedIps[rules->bannedIpCount], "\n")] = 0; rules->bannedIps[rules->bannedIpCount + 1] = nullptr; rules->bannedIpCount++; - printf("[Bans=%6lu] %15s matched pattern \"%s\": %s\n", rules->bannedIpCount, ip, rules->patterns[i], line); + if (print_bans) + printf("[Bans=%6lu] %15s matched pattern %4d (\"%s\"): %s\n", rules->bannedIpCount, ip, i, rules->patterns[i], line); - - signal(SIGCHLD, nullptr); __pid_t pid = fork(); if (pid == 0) { @@ -259,32 +283,90 @@ void process_line(char* line, void* cbData) perror("Fork failed"); exit(1); } + else + { + signal(SIGCHLD, SIG_IGN); + } return; } } + if (print_indeterminate) + printf("[Indeterminate]: %s\n", line); + free(ip); } - -int main(void) +int main(int argc, char* argv[]) { printf("Hello, World!\n"); + print_allowed_ip = getenv("D_PRINT_ALLOWED_IP") != NULL && strcmp(getenv("D_PRINT_ALLOWED_IP"), "1") == 0; print_allowed_pattern = getenv("D_PRINT_ALLOWED_PATTERN") != NULL && strcmp(getenv("D_PRINT_ALLOWED_PATTERN"), "1") == 0; - print_banned_ip = getenv("D_PRINT_BANNED_IP") != NULL && strcmp(getenv("D_PRINT_BANNED_IP"), "1") == 0; + print_already_banned_ip = getenv("D_PRINT_BANNED_IP") != NULL && strcmp(getenv("D_PRINT_BANNED_IP"), "1") == 0; + + for (int i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--allowed-ips") == 0 && i + 1 < argc) + allowed_ips_path = argv[++i]; + else if (strcmp(argv[i], "--allowed-patterns") == 0 && i + 1 < argc) + allowed_patterns_path = argv[++i]; + else if (strcmp(argv[i], "--patterns") == 0 && i + 1 < argc) + patterns_path = argv[++i]; + else if (strcmp(argv[i], "--access-log") == 0 && i + 1 < argc) + access_log_path = argv[++i]; + else if (strstr(argv[i], "--print-allowed-ip") != NULL) + print_allowed_ip = strstr(argv[i], "=false") == NULL; + else if (strstr(argv[i], "--print-allowed-pattern") != NULL) + print_allowed_pattern = strstr(argv[i], "=false") == NULL; + else if (strstr(argv[i], "--print-already-banned-ip") != NULL) + print_already_banned_ip = strstr(argv[i], "=false") == NULL; + else if (strstr(argv[i], "--print-bans") != NULL) + print_bans = strstr(argv[i], "=false") == NULL; + else if (strstr(argv[i], "--print-indeterminate") != NULL) + print_indeterminate = strstr(argv[i], "=false") == NULL; + else if (strcmp(argv[i], "--debug") == 0) + { + print_allowed_ip = true; + print_allowed_pattern = true; + print_already_banned_ip = true; + print_indeterminate = true; + print_bans = true; + } + else + { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + return 1; + } + } + + if (strcmp(access_log_path, "stdin") == 0 || strcmp(access_log_path, "-") == 0) + { + access_log_path = "/dev/stdin"; + } + + fprintf(stderr, "allowed_patterns_path: %s\n", allowed_patterns_path); + fprintf(stderr, "allowed_ips_path: %s\n", allowed_ips_path); + fprintf(stderr, "patterns_path: %s\n", patterns_path); + fprintf(stderr, "access_log_path: %s\n", access_log_path); + + fprintf(stderr, "print_allowed_ip: %hhd\n", print_allowed_ip); + fprintf(stderr, "print_allowed_pattern: %hhd\n", print_allowed_pattern); + fprintf(stderr, "print_already_banned_ip: %hhd\n", print_already_banned_ip); + fprintf(stderr, "print_indeterminate: %hhd\n", print_indeterminate); + fprintf(stderr, "print_bans: %hhd\n", print_bans); MatchRules rules; - rules.patterns = read_lines(PATTERNS_FILE); - rules.allowed_patterns = read_lines(ALLOWED_PATTERNS_FILE); - rules.allowedIps = read_lines(ALLOWED_IPS_FILE); - rules.bannedIpSize = 64; + rules.patterns = read_lines(patterns_path); + rules.allowed_patterns = read_lines(allowed_patterns_path); + rules.allowedIps = read_lines(allowed_ips_path); + rules.bannedIpSize = 8; rules.bannedIpCount = 0; rules.bannedIps = xmalloc(sizeof(char**) * rules.bannedIpSize); //rules.bannedIps[0] = ; - read_lines_cb(ACCESS_LOG_FILE, process_line, &rules); + read_lines_cb(access_log_path, process_line, &rules); for (int i = 0; rules.bannedIps[i] != NULL; i++) { diff --git a/p_user_agent_legacy_browser.txt b/p_user_agent_legacy_browser.txt deleted file mode 100644
index 87f53a0..0000000 --- a/p_user_agent_legacy_browser.txt +++ /dev/null
@@ -1,3 +0,0 @@ - Firefox/71.0" - Chrome/74.0 - Chrome/60.0 diff --git a/p_user_agent_legacy_win.txt b/p_user_agent_legacy_win.txt deleted file mode 100644
index e69de29..0000000 --- a/p_user_agent_legacy_win.txt +++ /dev/null
diff --git a/patterns.txt b/patterns.txt new file mode 100644
index 0000000..97ff2bc --- /dev/null +++ b/patterns.txt
@@ -0,0 +1,878 @@ +\x00 +\x01 +\x02 +\x03 +\x04 +\x05 +\x06 +\x07 +\x08 +\x09 +\x0A +\x0B +\x0C +\x0D +\x0E +\x0F +\x10 +\x11 +\x12 +\x13 +\x14 +\x15 +\x16 +\x17 +\x18 +\x19 +\x1A +\x1B +\x1C +\x1D +\x1E +\x1F +\x20 +\x21 +\x22 +\x23 +\x24 +\x25 +\x26 +\x27 +\x28 +\x29 +\x2A +\x2B +\x2C +\x2D +\x2E +\x2F +\x30 +\x31 +\x32 +\x33 +\x34 +\x35 +\x36 +\x37 +\x38 +\x39 +\x3A +\x3B +\x3C +\x3D +\x3E +\x3F +\x40 +\x41 +\x42 +\x43 +\x44 +\x45 +\x46 +\x47 +\x48 +\x49 +\x4A +\x4B +\x4C +\x4D +\x4E +\x4F +\x50 +\x51 +\x52 +\x53 +\x54 +\x55 +\x56 +\x57 +\x58 +\x59 +\x5A +\x5B +\x5C +\x5D +\x5E +\x5F +\x60 +\x61 +\x62 +\x63 +\x64 +\x65 +\x66 +\x67 +\x68 +\x69 +\x6A +\x6B +\x6C +\x6D +\x6E +\x6F +\x70 +\x71 +\x72 +\x73 +\x74 +\x75 +\x76 +\x77 +\x78 +\x79 +\x7A +\x7B +\x7C +\x7D +\x7E +\x7F +\x80 +\x81 +\x82 +\x83 +\x84 +\x85 +\x86 +\x87 +\x88 +\x89 +\x8A +\x8B +\x8C +\x8D +\x8E +\x8F +\x90 +\x91 +\x92 +\x93 +\x94 +\x95 +\x96 +\x97 +\x98 +\x99 +\x9A +\x9B +\x9C +\x9D +\x9E +\x9F +\xA0 +\xA1 +\xA2 +\xA3 +\xA4 +\xA5 +\xA6 +\xA7 +\xA8 +\xA9 +\xAA +\xAB +\xAC +\xAD +\xAE +\xAF +\xB0 +\xB1 +\xB2 +\xB3 +\xB4 +\xB5 +\xB6 +\xB7 +\xB8 +\xB9 +\xBA +\xBB +\xBC +\xBD +\xBE +\xBF +\xC0 +\xC1 +\xC2 +\xC3 +\xC4 +\xC5 +\xC6 +\xC7 +\xC8 +\xC9 +\xCA +\xCB +\xCC +\xCD +\xCE +\xCF +\xD0 +\xD1 +\xD2 +\xD3 +\xD4 +\xD5 +\xD6 +\xD7 +\xD8 +\xD9 +\xDA +\xDB +\xDC +\xDD +\xDE +\xDF +\xE0 +\xE1 +\xE2 +\xE3 +\xE4 +\xE5 +\xE6 +\xE7 +\xE8 +\xE9 +\xEA +\xEB +\xEC +\xED +\xEE +\xEF +\xF0 +\xF1 +\xF2 +\xF3 +\xF4 +\xF5 +\xF6 +\xF7 +\xF8 +\xF9 +\xFA +\xFB +\xFC +\xFD +\xFE +\xFF +;/actuator +;/env +;/internal +;/META-INF +/phpunit +eval-stdin.php +phpinfo +/.git/ +.env +.htaccess +web.config +/.svn/ +/.AWS_/credentials +/.DS_Store +/.__info.php +/.aws/ +/.circleci/configs/development.yml +/.config/sftp.json +/.dockerignore +/.npmrc +/.sendgrid +/.travis.yml +/.vscode/sftp.json + /%61%63%74%75%61%74%6f%72/%65%6e%76 + /%61%70%69/%61%63%74%75%61%74%6f%72/%65%6e%37%36 + /%61%70%69/%61%63%74%75%61%74%6f%72/%65%6e%76 + /%61%70%69/%65%6e%37%36 + /%61%70%69/%65%6e%76 + /%61%70%69/%69%6e%74%65%72%6e%61%6c/%61%63%74%75%61%74%6f%72/%65%6e%76 + /%65%6e%76 + /%67%61%74%65%77%61%79/%61%63%74%75%61%74%6f%72/%65%6e%76 + /%67%61%74%65%77%61%79/%65%6e%76 + /%6d%61%6e%61%67%65%6d%65%6e%74/%61%63%74%75%61%74%6f%72/%65%6e%76 + /%6d%61%6e%61%67%65%6d%65%6e%74/%65%6e%76 + /%6d%61%6e%61%67%65/%61%63%74%75%61%74%6f%72/%65%6e%76 + /%6d%61%6e%61%67%65/%65%6e%76 + /+CSCOE+/logon.html + /.wp-config.php.swp + //admin/login.asp + //installer.php + //webpages/login.html + /0-info.php + /00_server_info.php + /01-info.php + /0_info.php + /0info.php + /1.php + /1_1_PhpInfo.php + /2018/wp-includes/wlwmanifest.xml + /2019/wp-includes/wlwmanifest.xml + /3/3/3/3/3/3/3/3/3/3/3/3/3/3/3/3 + /5info.php + /@vite/env + /Account/Login + /AwsConfig.json + /Awsconfig.json + /CFIDE/componentutils/ + /CHANGELOG.txt + /CSS/Miniweb.css + /Dockerfile + /HNAP1 + /IPCamDesc.xml + /NuLM + /OA_HTML/AppsLocalLogin.jsp + /PHPInfo.php + /PSIA/index + /PhpInfo.php + /Portal/Portal.mwsl + /Portal0000.htm + /Public/home/js/check.js + /RDWeb/ + /README.md + /ReportServer + /Telerik.Web.UI.WebResource.axd + /WebApp/js/UI_String.js + /WebInterface/ + /__Additional + /_all_dbs + /_phpinf.php + /_wpeprivate/config.json + /aaa9 + /aab8 + /aab9 + /ab2g + /ab2h + /about + /actuator/env + /actuator/gateway/routes + /actuator/health + /admin.asp + /admin.cfm + /admin.cgi + /admin.html + /admin.jsa + /admin.jsp + /admin.php + /admin.pl + /admin/ + /administrator/index.php + /administrator/manifests/files/joomla.xml + /ads.txt + /alive.php + /allversions + /api-docs + /api/actuator/env + /api/config.js + /api/config/config.yml + /api/credentials + /api/env + /api/objects/codes.php.save + /api/proxy + /api/server/version + /api/session/properties + /api/sonicos/auth + /api/sonicos/is-sslvpn-enabled + /api/sonicos/tfa + /api/v1.0/environment + /api/v1/check-version + /api/v1/label/__name__/values + /api/v1/metadata + /api/v1/notifications/live + /api/v1/proxy + /api/v1/query + /api/v1/status/buildinfo + /api/v1/status/runtimeinfo + /api/v1/ws/server + /api/v2/about + /api/v2/proxy + /api/v3/meta + /api/version + /api/vip/i18n/api/v2/translation/products/vRNIUI/versions/1 + /api_keys/sendgrid_keys.json + /apis/config/config.js + /apis/controllers/users.js + /app.js + /app.py + /app/config/parameters.yml + /app/etc/env.php + /app/etc/local.xml + /application.properties + /application.yml + /apps/zxtm/login.cgi + /aspera/faspex/ + /assets/env.js + /assets/index-BPbBbNOr.css + /assets/index-DuE_NgAI.js + /aura + /auth.html + /auth.json + /auth1.html + /autodiscover/autodiscover.json + /aws-secret.yaml + /aws.yml + /aws/credentials + /backend/config/default.yml + /backend/config/development.yml + /backend/config/settings.yml + /backup + /base.cfm + /base.jsa + /base.shtml + /baseDstu2/metadata + /baseDstu3/metadata + /baseR2/metadata + /baseR3/metadata + /baseR4/metadata + /baseR5/metadata + /bitrix/php_interface/dbconn.php + /blog + /boaform/admin/formLogin + /bootstrap/cache/config.php + /build.gradle + /c/login + /cdn-cgi/trace/cdn-cgi/trace + /centreon/api/latest/platform/versions + /cf_scripts/scripts/ajax/ckeditor/ckeditor.js + /cgi-bin/ + /cloud/Scraper.js + /cluster/list.query + /cms/wp-includes/wlwmanifest.xml + /composer.json + /computeMetadata/v1 + /config + /confluence/rest/applinks/1.0/manifest + /contact/ + /containers/json + /controller/admin/post.js + /controller/api/post.js + /controllers/settings.js + /core/install.php + /cslu/v1/core/conf + /css/elfinder.min.css + /css/eonweb.css + /css/images/PTZOptics_powerby.png + /dana-cached/hc/HostCheckerInstaller.osx + /dana-na/nc/nc_gina_ver.txt + /database.json + /db.ini + /db_backup.sql + /debug.php + /debug/default/view + /default.html + /default.jhtml + /default.jsa + /developmentserver/metadatauploader + /dniapi/userInfos + /dns-query + /doc/index.html + /docker-compose. + /docs/cplugError.html/ + /druid/index.html + /ecp/Current/exporttool/microsoft.exchange.ediscovery.exporttool.application + /en%76; + /env + /epa/scripts/win/nsepa_setup.exe + /etc/gitconfig + /evox/about + /ext-js/app/common/zld_product_spec.js + /fhir-server/api/v4/metadata + /fhir/metadata + /fog/management/index.php + /form.html + /forms/doLogin + /ftptest.cgi + /gatsby-config.js + /geoip/ + /geoserver + /getcpuutil.php-bakworking + /git/.config + /global-protect/login.esp + /health + /hello.world + /helm/values.yaml + /helpdesk/WebObjects/Helpdesk.woa + /helper.js + /helper/EmailHelper.js + /helpers/utility.js + /home.aspx + /home.cfm + /home.html + /hudson + /human.aspx + /i.php + /identity + /index.aspx + /index.cfm + /index.cgi + /index.jsp + /index.php + /index.pl + /index.shtml + /indice.cfm + /indice.cgi + /indice.html + /info + /inicio.php + /inicio.pl + /instance/sendgrid_keys.py + /internal/api + /internal/proxy + /internal_forms_authentication + /jasperserver-pro/login.html + /jasperserver/login.html + /jasperserverTest/login.html + /javascript/validation/OEM.js + /joomla/configuration.php-dist + /js/NewWindow_2_all.js + /js/app.js + /js/config.js + /js/main.js + /karma.conf.json + /keys/sendgrid_keys.json + /kylin/ + /language/en-GB/en-GB.xml + /lara/info.php + /laravel/info.php + /latest/meta-data + /latest/user-data + /lms/db + /local.inc.php + /local.ini + /local.json + /local.php + /local.xml + /local_settings.py + /localstart.jsa + /login.action + /login.asp + /login.do + /login.htm + /login.jsp + /login.php + /login/login.html + /logon/LogonPoint/index.html + /logs/error.php + /magento_version + /main.asp + /main.cfm + /main.js + /main.yml + /manage/account/login + /manager/html + /media/wp-includes/wlwmanifest.xml + /menu.aspx + /menu.cfm + /menu.php + /metadata + /modules/contrib/sendgrid_mail/ + /my_env/chakaash.py + /my_env/newsletter.py + /my_env/palash.py + /mytest/astech_robot.js + /new + /nmaplowercheck1745842941 + /nova-api/styles + /o8kJ + /odinhttpcall1746060903 + /odinhttpcall1746102992 + /officescan/console/cgi/cgiChkMasterPwd.exe + /officescan/console/html/localization.js + /old + /onvif/device_service + /openapi.json + /openapi/v2 + /opt/aws/ + /owa/ + /owncloud/status.php + /p/login/ + /package.json + /pandora_console/ + /parameters.yaml + /parameters.yml + /partner/config/config.js + /partymgr/control/main + /password.php + /php.ini + /php.php + /php/ztp_gate.php/.js.map + /php_info + /phpmyadmin/index.php + /phpmyadmin4.8.5/index.php + /pi.php + /pmd/index.php + /pom.xml + /pools + /portal + /private/ + /proxy + /public/index.php + /public/js/main.js + /query + /r-seenet/index.php + /r2/metadata + /r3/metadata + /r4/metadata + /r5/metadata + /readme.txt + /register/ + /remote/fgt_lang + /remote/login + /resolve + /rest/applinks/1.0/manifest + /root/info.php + /root/infophp + /s/aura + /s/fact + /s/sfsites/aura + /s3.js + /scripts/WPnBr.dll + /scripts/nodemailer.js + /sdk + /secret + /secure-config.json + /sendgrid + /server-info + /server-status + /server.js + /server.php + /server/config/database.js + /server/s3.js + /server_info.php + /service/email_service.py + /settings.bak + /settings.cfg + /settings.inc.php + /settings.ini + /settings.json + /settings.php + /settings.py + /settings.xml + /settings.yaml + /settings/sendgrid_config + /settings/sendgrid_keys + /setup.cgi + /sfsites/aura + /sftp-config.json + /sftp.json + /shared/config/config.js + /shop/wp-includes/wlwmanifest.xml + /showLogin.cc + /site/wp-includes/wlwmanifest.xml + /sitecore/shell/sitecore.version.xml + /sitemap.xml + /sites/all/modules/ + /sites/all/themes/ + /sites/default/settings.php + /sito/wp-includes/wlwmanifest.xml + /sms.py + /solr/ + /sonicui/7/login/ + /src/config.js + /sslvpnLogin.html + /sslvpn_logon.shtml + /static/admin/javascript/hetong.js + /static/historypage.js + /stats + /status + /storage/framework/cache/ + /storage/framework/sessions/ + /storage/framework/views/ + /storage/logs/laravel.log + /storage/sendgrid.json + /sugar_version.json + /swagger-ui.html + /swagger.js + /systembc/password.php + /t4 + /telescope/requests + /temp + /teorema505 + /test + /tmp.php + /tos/index.php + /typo3conf/localconf.php + /ucT0 + /upl.php + /user + /v1 + /v2/_catalog + /var/aws/ + /var/lib/aws/ + /var/log/ + /var/logs/ + /versa/login + /version + /vpn/index.html + /vpnsvc/connect.cgi + /wRGL + /web/ + /webportal.cgi + /website/wp-includes/wlwmanifest.xml + /webui + /wordpress + /workplace/home.action + /wp + /wsman + /xml/info.xml + /xmldata + /xmlrpc.php + /yarn.lock + /zabbix/favicon.ico + <NULL> + "CONNECT +# OpenAI ++https://openai.com/gptbot +OAI-SearchBot/ +# Claude ++claudebot@anthropic.com +ClaudeBot/1.0 +AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com) +Mozilla/5.0 zgrab/ +Mozilla/5.0; Keydrop.io/ +FH Muenster/Security-Scanner/fh-muenster.de +"'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/genomecrawler)'" +l9explore/1.2.2 +"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" +"Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)" + abuse.xmco.fr" ++https://developer.amazon.com/support/amazonbot +http://mj12bot.com/ +SeznamBot/ +SemanticScholarBot +NetcraftSurveyAgent +ALittle Client +scaninfo@paloaltonetworks.com +Scrapy/ +CensysInspect/ +ModatScanner/ +AppleBot +AhrefsBot + Chrome/10. + Chrome/11. + Chrome/12. + Chrome/13. + Chrome/14. + Chrome/15. + Chrome/16. + Chrome/17. + Chrome/18. + Chrome/19. + Chrome/20. + Chrome/21. + Chrome/22. + Chrome/23. + Chrome/24. + Chrome/25. + Chrome/26. + Chrome/27. + Chrome/28. + Chrome/29. + Chrome/3 + Chrome/4 + Chrome/5 + Chrome/6 + Chrome/7 + Chrome/8 + Chrome/9 + CriOS/10. + CriOS/11. + CriOS/12. + CriOS/13. + CriOS/14. + CriOS/15. + CriOS/16. + CriOS/17. + CriOS/18. + CriOS/19. + CriOS/20. + CriOS/21. + CriOS/22. + CriOS/23. + CriOS/24. + CriOS/25. + CriOS/26. + CriOS/27. + CriOS/28. + CriOS/29. + CriOS/3 + CriOS/4 + CriOS/5 + CriOS/6 + CriOS/7 + CriOS/8 + CriOS/9 + Firefox/10. + Firefox/11. + Firefox/12. + Firefox/13. + Firefox/14. + Firefox/15. + Firefox/16. + Firefox/17. + Firefox/18. + Firefox/19. + Firefox/20. + Firefox/21. + Firefox/22. + Firefox/23. + Firefox/24. + Firefox/25. + Firefox/26. + Firefox/27. + Firefox/28. + Firefox/29. + Firefox/3 + Firefox/4 + Firefox/5 + Firefox/6 + Firefox/7 + Firefox/8 + Firefox/9 + FxiOS/10. + FxiOS/11. + FxiOS/12. + FxiOS/13. + FxiOS/14. + FxiOS/15. + FxiOS/16. + FxiOS/17. + FxiOS/18. + FxiOS/19. + FxiOS/20. + FxiOS/21. + FxiOS/22. + FxiOS/23. + FxiOS/24. + FxiOS/25. + FxiOS/26. + FxiOS/27. + FxiOS/28. + FxiOS/29. + FxiOS/3 + FxiOS/4 + FxiOS/5 + FxiOS/6 + FxiOS/7 + FxiOS/8 + FxiOS/9 + Presto/ + Trident/ +PPC Mac OS X +Win 9x +Windows CE +Windows 95 +Windows 98 +Windows NT 4 +Windows NT 5 +Windows NT 6 +Windows NT 10.0; Trident +CPU iPhone OS 3 +CPU iPhone OS 4 +CPU iPhone OS 5 +CPU iPhone OS 6 +CPU iPhone OS 7 +CPU iPhone OS 8 +CPU iPhone OS 9 +Android 2. +Android 3. +Android 4. +Android 5. +Android 6. +MSIE 6.0 diff --git a/template/ai_emma.txt b/template/ai_emma.txt new file mode 100644
index 0000000..849702f --- /dev/null +++ b/template/ai_emma.txt
@@ -0,0 +1,4 @@ +109.128.185.4 +192.168. +127. +10. diff --git a/ap_emma.txt b/template/ap_emma.txt
index a1ff752..a1ff752 100644 --- a/ap_emma.txt +++ b/template/ap_emma.txt
diff --git a/ap_matrix.txt b/template/ap_matrix.txt
index 7896936..b422bd3 100644 --- a/ap_matrix.txt +++ b/template/ap_matrix.txt
@@ -1,3 +1,5 @@ /.well-known/matrix/ /_matrix/client/ /_matrix/federation/ + /_matrix/key + diff --git a/template/p_escaped_hex.txt b/template/p_escaped_hex.txt new file mode 100644
index 0000000..42fd805 --- /dev/null +++ b/template/p_escaped_hex.txt
@@ -0,0 +1,256 @@ +\x00 +\x01 +\x02 +\x03 +\x04 +\x05 +\x06 +\x07 +\x08 +\x09 +\x0A +\x0B +\x0C +\x0D +\x0E +\x0F +\x10 +\x11 +\x12 +\x13 +\x14 +\x15 +\x16 +\x17 +\x18 +\x19 +\x1A +\x1B +\x1C +\x1D +\x1E +\x1F +\x20 +\x21 +\x22 +\x23 +\x24 +\x25 +\x26 +\x27 +\x28 +\x29 +\x2A +\x2B +\x2C +\x2D +\x2E +\x2F +\x30 +\x31 +\x32 +\x33 +\x34 +\x35 +\x36 +\x37 +\x38 +\x39 +\x3A +\x3B +\x3C +\x3D +\x3E +\x3F +\x40 +\x41 +\x42 +\x43 +\x44 +\x45 +\x46 +\x47 +\x48 +\x49 +\x4A +\x4B +\x4C +\x4D +\x4E +\x4F +\x50 +\x51 +\x52 +\x53 +\x54 +\x55 +\x56 +\x57 +\x58 +\x59 +\x5A +\x5B +\x5C +\x5D +\x5E +\x5F +\x60 +\x61 +\x62 +\x63 +\x64 +\x65 +\x66 +\x67 +\x68 +\x69 +\x6A +\x6B +\x6C +\x6D +\x6E +\x6F +\x70 +\x71 +\x72 +\x73 +\x74 +\x75 +\x76 +\x77 +\x78 +\x79 +\x7A +\x7B +\x7C +\x7D +\x7E +\x7F +\x80 +\x81 +\x82 +\x83 +\x84 +\x85 +\x86 +\x87 +\x88 +\x89 +\x8A +\x8B +\x8C +\x8D +\x8E +\x8F +\x90 +\x91 +\x92 +\x93 +\x94 +\x95 +\x96 +\x97 +\x98 +\x99 +\x9A +\x9B +\x9C +\x9D +\x9E +\x9F +\xA0 +\xA1 +\xA2 +\xA3 +\xA4 +\xA5 +\xA6 +\xA7 +\xA8 +\xA9 +\xAA +\xAB +\xAC +\xAD +\xAE +\xAF +\xB0 +\xB1 +\xB2 +\xB3 +\xB4 +\xB5 +\xB6 +\xB7 +\xB8 +\xB9 +\xBA +\xBB +\xBC +\xBD +\xBE +\xBF +\xC0 +\xC1 +\xC2 +\xC3 +\xC4 +\xC5 +\xC6 +\xC7 +\xC8 +\xC9 +\xCA +\xCB +\xCC +\xCD +\xCE +\xCF +\xD0 +\xD1 +\xD2 +\xD3 +\xD4 +\xD5 +\xD6 +\xD7 +\xD8 +\xD9 +\xDA +\xDB +\xDC +\xDD +\xDE +\xDF +\xE0 +\xE1 +\xE2 +\xE3 +\xE4 +\xE5 +\xE6 +\xE7 +\xE8 +\xE9 +\xEA +\xEB +\xEC +\xED +\xEE +\xEF +\xF0 +\xF1 +\xF2 +\xF3 +\xF4 +\xF5 +\xF6 +\xF7 +\xF8 +\xF9 +\xFA +\xFB +\xFC +\xFD +\xFE +\xFF diff --git a/p_path_contains.txt b/template/p_path_contains.txt
index b53129c..b53129c 100644 --- a/p_path_contains.txt +++ b/template/p_path_contains.txt
diff --git a/p_paths.txt b/template/p_paths.txt
index 9508814..9508814 100644 --- a/p_paths.txt +++ b/template/p_paths.txt
diff --git a/p_separate.txt b/template/p_separate.txt
index f7b9062..f7b9062 100644 --- a/p_separate.txt +++ b/template/p_separate.txt
diff --git a/p_user_agents.txt b/template/p_user_agent_bot.txt
index 0d25399..08ab337 100644 --- a/p_user_agents.txt +++ b/template/p_user_agent_bot.txt
@@ -1,3 +1,10 @@ +# OpenAI ++https://openai.com/gptbot +OAI-SearchBot/ +# Claude ++claudebot@anthropic.com +ClaudeBot/1.0 + AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com) Mozilla/5.0 zgrab/ Mozilla/5.0; Keydrop.io/ @@ -7,3 +14,15 @@ l9explore/1.2.2 "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" "Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)" abuse.xmco.fr" ++https://developer.amazon.com/support/amazonbot +http://mj12bot.com/ +SeznamBot/ +SemanticScholarBot +NetcraftSurveyAgent +ALittle Client +scaninfo@paloaltonetworks.com +Scrapy/ +CensysInspect/ +ModatScanner/ +AppleBot +AhrefsBot diff --git a/template/p_user_agent_legacy_browser.txt b/template/p_user_agent_legacy_browser.txt new file mode 100644
index 0000000..a318b0c --- /dev/null +++ b/template/p_user_agent_legacy_browser.txt
@@ -0,0 +1,110 @@ + Chrome/10. + Chrome/11. + Chrome/12. + Chrome/13. + Chrome/14. + Chrome/15. + Chrome/16. + Chrome/17. + Chrome/18. + Chrome/19. + Chrome/20. + Chrome/21. + Chrome/22. + Chrome/23. + Chrome/24. + Chrome/25. + Chrome/26. + Chrome/27. + Chrome/28. + Chrome/29. + Chrome/3 + Chrome/4 + Chrome/5 + Chrome/6 + Chrome/7 + Chrome/8 + Chrome/9 + CriOS/10. + CriOS/11. + CriOS/12. + CriOS/13. + CriOS/14. + CriOS/15. + CriOS/16. + CriOS/17. + CriOS/18. + CriOS/19. + CriOS/20. + CriOS/21. + CriOS/22. + CriOS/23. + CriOS/24. + CriOS/25. + CriOS/26. + CriOS/27. + CriOS/28. + CriOS/29. + CriOS/3 + CriOS/4 + CriOS/5 + CriOS/6 + CriOS/7 + CriOS/8 + CriOS/9 + Firefox/10. + Firefox/11. + Firefox/12. + Firefox/13. + Firefox/14. + Firefox/15. + Firefox/16. + Firefox/17. + Firefox/18. + Firefox/19. + Firefox/20. + Firefox/21. + Firefox/22. + Firefox/23. + Firefox/24. + Firefox/25. + Firefox/26. + Firefox/27. + Firefox/28. + Firefox/29. + Firefox/3 + Firefox/4 + Firefox/5 + Firefox/6 + Firefox/7 + Firefox/8 + Firefox/9 + FxiOS/10. + FxiOS/11. + FxiOS/12. + FxiOS/13. + FxiOS/14. + FxiOS/15. + FxiOS/16. + FxiOS/17. + FxiOS/18. + FxiOS/19. + FxiOS/20. + FxiOS/21. + FxiOS/22. + FxiOS/23. + FxiOS/24. + FxiOS/25. + FxiOS/26. + FxiOS/27. + FxiOS/28. + FxiOS/29. + FxiOS/3 + FxiOS/4 + FxiOS/5 + FxiOS/6 + FxiOS/7 + FxiOS/8 + FxiOS/9 + Presto/ + Trident/ diff --git a/template/p_user_agent_legacy_os.txt b/template/p_user_agent_legacy_os.txt new file mode 100644
index 0000000..ea58417 --- /dev/null +++ b/template/p_user_agent_legacy_os.txt
@@ -0,0 +1,22 @@ +PPC Mac OS X +Win 9x +Windows CE +Windows 95 +Windows 98 +Windows NT 4 +Windows NT 5 +Windows NT 6 +Windows NT 10.0; Trident +CPU iPhone OS 3 +CPU iPhone OS 4 +CPU iPhone OS 5 +CPU iPhone OS 6 +CPU iPhone OS 7 +CPU iPhone OS 8 +CPU iPhone OS 9 +Android 2. +Android 3. +Android 4. +Android 5. +Android 6. +MSIE 6.0