Deduplicate patterns, fix dedup.c
2 files changed, 6 insertions, 9 deletions
diff --git a/dedup.c b/dedup.c
index fdfce87..36878c4 100644
--- a/dedup.c
+++ b/dedup.c
@@ -104,7 +104,7 @@ char** grow_array(char** array, size_t newSize)
void check_dups(char* filename) {
printf("# %s:\n", filename);
- char** lines = read_lines(PATTERNS_FILE);
+ char** lines = read_lines(filename);
if (lines == NULL)
{
diff --git a/template/p_user_agent_bot.txt b/template/p_user_agent_bot.txt
index 543adbb..c177145 100644
--- a/template/p_user_agent_bot.txt
+++ b/template/p_user_agent_bot.txt
@@ -4,15 +4,12 @@
OAI-SearchBot/
# Claude
+claudebot@anthropic.com
-ClaudeBot/1.0
+ClaudeBot/
# Facebook
-facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
-facebookexternalhit/1.1
-facebookcatalog/1.0
-meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)
-meta-externalagent/1.1
-meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)
-meta-externalfetcher/1.1
+facebookexternalhit/
+facebookcatalog/
+meta-externalagent/
+meta-externalfetcher/
# Other (by linked help domain)
+http://yandex.com
+https://developer.amazon.com
|
