summary refs log tree commit diff
path: root/template
diff options
context:
space:
mode:
authorRory& <root@rory.gay>2025-06-05 05:55:45 +0200
committerRory& <root@rory.gay>2025-06-05 05:55:45 +0200
commitb13161b321d43c31d2c126e38a96d3deee5a70ef (patch)
treeaa92c473f58844330583f3e43aed02433aa0e71d /template
parentPattern update, add cli flags (diff)
downloadBlockScrapersC-b13161b321d43c31d2c126e38a96d3deee5a70ef.tar.xz
Changes
Diffstat (limited to 'template')
-rw-r--r--template/ap_emma.txt5
-rw-r--r--template/ap_matrix.txt2
-rw-r--r--template/ap_owncast.txt8
-rw-r--r--template/p_paths.txt1
-rw-r--r--template/p_separate.txt3
-rw-r--r--template/p_user_agent_bot.txt26
-rw-r--r--template/p_user_agent_legacy_browser.txt40
-rw-r--r--template/p_user_agent_legacy_os.txt39
-rw-r--r--template/p_user_agent_likely_fake.txt3
-rw-r--r--template/p_user_agent_unknown.txt1
10 files changed, 95 insertions, 33 deletions
diff --git a/template/ap_emma.txt b/template/ap_emma.txt

index a1ff752..2412087 100644 --- a/template/ap_emma.txt +++ b/template/ap_emma.txt
@@ -1,2 +1,5 @@ GET /server.git/ -GET /matrix/thirdparty/nheko.git/ \ No newline at end of file +GET /matrix/thirdparty/nheko.git/ +# git: +"-" "git/ +"Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" \ No newline at end of file diff --git a/template/ap_matrix.txt b/template/ap_matrix.txt
index b422bd3..c1349d9 100644 --- a/template/ap_matrix.txt +++ b/template/ap_matrix.txt
@@ -2,4 +2,4 @@ /_matrix/client/ /_matrix/federation/ /_matrix/key - +"Synapse (bot; +https://github.com/matrix-org/synapse)" \ No newline at end of file diff --git a/template/ap_owncast.txt b/template/ap_owncast.txt new file mode 100644
index 0000000..8107640 --- /dev/null +++ b/template/ap_owncast.txt
@@ -0,0 +1,8 @@ +# Get stream metadata +"GET /hls/stream.m3u8 +"GET /hls/0/stream.m3u8 +# mpv +"-" "libmpv" +# initiated by... is this too loose? + "https://stream.rory.gay/sw.js" + "https://stream.rory.gay/" \ No newline at end of file diff --git a/template/p_paths.txt b/template/p_paths.txt
index 9508814..59cf8d1 100644 --- a/template/p_paths.txt +++ b/template/p_paths.txt
@@ -437,3 +437,4 @@ /yarn.lock /zabbix/favicon.ico <NULL> + //recordings/theme/main.css \ No newline at end of file diff --git a/template/p_separate.txt b/template/p_separate.txt
index f7b9062..a787b04 100644 --- a/template/p_separate.txt +++ b/template/p_separate.txt
@@ -1 +1,4 @@ "CONNECT + "SSTP_DUPLEX_POST +# Why are you doing direct connections? +{host="51.210.113.110" \ No newline at end of file diff --git a/template/p_user_agent_bot.txt b/template/p_user_agent_bot.txt
index 08ab337..543adbb 100644 --- a/template/p_user_agent_bot.txt +++ b/template/p_user_agent_bot.txt
@@ -1,20 +1,34 @@ # OpenAI +https://openai.com/gptbot ++https://openai.com/bot OAI-SearchBot/ # Claude +claudebot@anthropic.com ClaudeBot/1.0 - -AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com) +# Facebook +facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php) +facebookexternalhit/1.1 +facebookcatalog/1.0 +meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler) +meta-externalagent/1.1 +meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler) +meta-externalfetcher/1.1 +# Other (by linked help domain) ++http://yandex.com ++https://developer.amazon.com ++https://search.brave.com ++https://opensiteexplorer.org +# Other (mozilla/5.0) Mozilla/5.0 zgrab/ Mozilla/5.0; Keydrop.io/ -FH Muenster/Security-Scanner/fh-muenster.de "'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/genomecrawler)'" -l9explore/1.2.2 "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" "Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)" +# "Security scanners" +AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com) +FH Muenster/Security-Scanner/fh-muenster.de +l9explore/1.2.2 abuse.xmco.fr" -+https://developer.amazon.com/support/amazonbot http://mj12bot.com/ SeznamBot/ SemanticScholarBot @@ -26,3 +40,5 @@ CensysInspect/ ModatScanner/ AppleBot AhrefsBot +# Unknown and nondescript + "link_checker/0.1.0" diff --git a/template/p_user_agent_legacy_browser.txt b/template/p_user_agent_legacy_browser.txt
index a318b0c..a882dc0 100644 --- a/template/p_user_agent_legacy_browser.txt +++ b/template/p_user_agent_legacy_browser.txt
@@ -1,6 +1,6 @@ - Chrome/10. - Chrome/11. - Chrome/12. + Chrome/10 + Chrome/11 + Chrome/12 Chrome/13. Chrome/14. Chrome/15. @@ -8,16 +8,7 @@ Chrome/17. Chrome/18. Chrome/19. - Chrome/20. - Chrome/21. - Chrome/22. - Chrome/23. - Chrome/24. - Chrome/25. - Chrome/26. - Chrome/27. - Chrome/28. - Chrome/29. + Chrome/2 Chrome/3 Chrome/4 Chrome/5 @@ -52,9 +43,9 @@ CriOS/7 CriOS/8 CriOS/9 - Firefox/10. - Firefox/11. - Firefox/12. + Firefox/10 + Firefox/11 + Firefox/12 Firefox/13. Firefox/14. Firefox/15. @@ -62,16 +53,7 @@ Firefox/17. Firefox/18. Firefox/19. - Firefox/20. - Firefox/21. - Firefox/22. - Firefox/23. - Firefox/24. - Firefox/25. - Firefox/26. - Firefox/27. - Firefox/28. - Firefox/29. + Firefox/2 Firefox/3 Firefox/4 Firefox/5 @@ -108,3 +90,9 @@ FxiOS/9 Presto/ Trident/ + Opera 1. + Opera 2. + Opera 3. + Opera 4. + Opera 5. + Opera 6. \ No newline at end of file diff --git a/template/p_user_agent_legacy_os.txt b/template/p_user_agent_legacy_os.txt
index ea58417..30966ba 100644 --- a/template/p_user_agent_legacy_os.txt +++ b/template/p_user_agent_legacy_os.txt
@@ -1,8 +1,22 @@ PPC Mac OS X +Mac OS X 10_1_ +Mac OS X 10_2_ +Mac OS X 10_3 +Mac OS X 10_4 +Mac OS X 10_5 +Mac OS X 10_6 +Mac OS X 10_7 +Mac OS X 10_8 +Mac OS X 10_9 +Mac OS X 10_10 +Mac OS X 10_11 +Mac OS X 10_12 +Mac OS X 10_13 Win 9x Windows CE Windows 95 Windows 98 +Windows XP Windows NT 4 Windows NT 5 Windows NT 6 @@ -14,9 +28,34 @@ CPU iPhone OS 6 CPU iPhone OS 7 CPU iPhone OS 8 CPU iPhone OS 9 +CPU iPhone OS 10 +CPU iPhone OS 11 +CPU iPhone OS 12 +CPU iPhone OS 13 +CPU iPhone OS 14 +CPU iPhone OS 15 +CPU iPhone OS 16 +CPU iPad OS 3 +CPU iPad OS 4 +CPU iPad OS 5 +CPU iPad OS 6 +CPU iPad OS 7 +CPU iPad OS 8 +CPU iPad OS 9 +CPU iPad OS 10 +CPU iPad OS 11 +CPU iPad OS 12 +CPU iPad OS 13 +CPU iPad OS 14 +CPU iPad OS 15 +CPU iPad OS 16 Android 2. Android 3. Android 4. Android 5. Android 6. +Android 7. +Android 8. +Android 9. +Android 10. MSIE 6.0 diff --git a/template/p_user_agent_likely_fake.txt b/template/p_user_agent_likely_fake.txt new file mode 100644
index 0000000..9e762fe --- /dev/null +++ b/template/p_user_agent_likely_fake.txt
@@ -0,0 +1,3 @@ +.0.0.0 +(Windows; U; Windows NT 10.0) AppleWebKit/ +(Windows; U; Windows NT 11.0) AppleWebKit/ \ No newline at end of file diff --git a/template/p_user_agent_unknown.txt b/template/p_user_agent_unknown.txt new file mode 100644
index 0000000..f937ab3 --- /dev/null +++ b/template/p_user_agent_unknown.txt
@@ -0,0 +1 @@ +"Linux Gnu (cow)" \ No newline at end of file