diff options
author | Eric Eastwood <erice@element.io> | 2021-10-29 17:27:38 -0500 |
---|---|---|
committer | Eric Eastwood <erice@element.io> | 2021-10-29 17:27:38 -0500 |
commit | e0ff66dfd9949d83d9be13f62faf96b68446065e (patch) | |
tree | d8e29de0d5c4be249fb1d4aa8ffcfbac99c2cc3d | |
parent | Implement backfill in handler so we can do fetching later (diff) | |
download | synapse-e0ff66dfd9949d83d9be13f62faf96b68446065e.tar.xz |
Fix backfill being able to cleanly branch into history and back to "live"
Fix https://github.com/matrix-org/synapse/pull/11114#discussion_r731363046 - When we see a connected insertion event indicating a branch of history, we now don't accidentally leak the base event we're branching from in the backfill chunk until all of the events in the historical brach are exhausted. - Backfill will now check whether the federating homeserver asking has a given insertion event when we see one connected to the DAG. This allows backfill to initially decide to give all the historical messages in the branch, then next time when the base event we branched from comes up in backfill again, it sees that federated homeserver already has the insertion event, and we can continue down the normal DAG prev_event path instead. Example Backfill response (in order yay): Overview: ``` eventIDsAfter batch0 batch1 eventIDsBefore ``` ``` new implementation backfill events(100)=[ "event_id=$q3w2dF5mWBkgUF1ADdVgS_bX7YQuqODJ2HC33bln4Ho,depth=11,body=Message 1 (eventIDsAfter),prevs=['$Com3-ZIR35FvYcBQvHdpxGsS1iW-DO_Z5x1zZ5R5Az8']", "event_id=$Com3-ZIR35FvYcBQvHdpxGsS1iW-DO_Z5x1zZ5R5Az8,depth=10,body=Message 0 (eventIDsAfter),prevs=['$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI']", "event_id=$78Yy5bqa9XuQ5fhcw9AF36Xf4W5EDcjsraSInif7nYU,depth=10,body=org.matrix.msc2716.insertion,prevs=['$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI']", "event_id=$GEzF4euV8StAtdCDCtWmpdhxRE3KQWgreyr1_ePqlpM,depth=10,body=org.matrix.msc2716.batch,prevs=['$ot97LEPqM55EV1ka8YM445c6q-ny4jBBRxjgQnGxihQ']", "event_id=$ot97LEPqM55EV1ka8YM445c6q-ny4jBBRxjgQnGxihQ,depth=10,body=Historical 99 (batch=0),prevs=['$i53GQczFiX-tqXgcxudp5AWR9C-A0yJiET_ZXe9D49I']", "event_id=$i53GQczFiX-tqXgcxudp5AWR9C-A0yJiET_ZXe9D49I,depth=10,body=Historical 98 (batch=0),prevs=['$lqjhSBLzbi2WfWNAIm1mngStqsPfQJUwPO0NAFtB8lo']", "event_id=$lqjhSBLzbi2WfWNAIm1mngStqsPfQJUwPO0NAFtB8lo,depth=10,body=Historical 97 (batch=0),prevs=['$k8OC791p_vB-Hz36Byn-o3vK9x_rc0qkVufBpxYRcaA']", "event_id=$k8OC791p_vB-Hz36Byn-o3vK9x_rc0qkVufBpxYRcaA,depth=10,body=Historical 96 (batch=0),prevs=['$9obrnkfP2HpDfKUkWyHrtHML5_31MZ3T9sKQOb6eUV8']", "event_id=$9obrnkfP2HpDfKUkWyHrtHML5_31MZ3T9sKQOb6eUV8,depth=10,body=Historical 95 (batch=0),prevs=['$PSNLMD1RhBvR9ffpbEOKtUHE9PUUiacJ2g8_DA9nSjM']", "event_id=$PSNLMD1RhBvR9ffpbEOKtUHE9PUUiacJ2g8_DA9nSjM,depth=10,body=Historical 94 (batch=0),prevs=['$vGkUiP049MLeapDGHegqdiGZRkuc6WxEJ-iOAN7mqr4']", "event_id=$vGkUiP049MLeapDGHegqdiGZRkuc6WxEJ-iOAN7mqr4,depth=10,body=Historical 93 (batch=0),prevs=['$reYXzhTXHlqqk6XMLK_mEIFQGIesn4NJ70JYa7EziQc']", "event_id=$reYXzhTXHlqqk6XMLK_mEIFQGIesn4NJ70JYa7EziQc,depth=10,body=Historical 92 (batch=0),prevs=['$dg8ZPPPqh1SbIhcg-GMvEGz49h53TUMoZ8yO4zbfA4U']", "event_id=$dg8ZPPPqh1SbIhcg-GMvEGz49h53TUMoZ8yO4zbfA4U,depth=10,body=Historical 91 (batch=0),prevs=['$QW7wied5SyHtJElJTo4SUkc-zNLlAh6tbCiAa5EJXg4']", "event_id=$QW7wied5SyHtJElJTo4SUkc-zNLlAh6tbCiAa5EJXg4,depth=10,body=Historical 90 (batch=0),prevs=['$COQuVmW8z0geXFZ2fPdo_kW0q1NK1LLZSI7fZNIwvgM']", "event_id=$COQuVmW8z0geXFZ2fPdo_kW0q1NK1LLZSI7fZNIwvgM,depth=10,body=Historical 89 (batch=0),prevs=['$04k2wOdm-L41JDQ73KPpDe1njG3L9FYjOsNVamFsDQ0']", "event_id=$04k2wOdm-L41JDQ73KPpDe1njG3L9FYjOsNVamFsDQ0,depth=10,body=Historical 88 (batch=0),prevs=['$70BsBGyeg4Pf3VhGhXjtk4N5XKNNSit4vjnrirwTmic']", "event_id=$70BsBGyeg4Pf3VhGhXjtk4N5XKNNSit4vjnrirwTmic,depth=10,body=Historical 87 (batch=0),prevs=['$Y_6AET6hMZY42rPki4r0GxMfs8E_TCJyepNL1Vfwnh4']", "event_id=$Y_6AET6hMZY42rPki4r0GxMfs8E_TCJyepNL1Vfwnh4,depth=10,body=Historical 86 (batch=0),prevs=['$jSk0bO6PFUcqxHsjTvFPyJZoXL4N9nEGex7I-yyvBj8']", "event_id=$jSk0bO6PFUcqxHsjTvFPyJZoXL4N9nEGex7I-yyvBj8,depth=10,body=Historical 85 (batch=0),prevs=['$wPV_BB23RIolkHz0yGTvwt1-TJbqpGPqZc33vR5DUMw']", "event_id=$wPV_BB23RIolkHz0yGTvwt1-TJbqpGPqZc33vR5DUMw,depth=10,body=Historical 84 (batch=0),prevs=['$X2FJ_Jmoy5sx9u0UWTN1bkds6diO0uDxxRGco4r9Msw']", "event_id=$X2FJ_Jmoy5sx9u0UWTN1bkds6diO0uDxxRGco4r9Msw,depth=10,body=Historical 83 (batch=0),prevs=['$jrmzKa3aDmpZg5HgDHQqQ_lgl9L4rDWiWk7xd1igfAE']", "event_id=$jrmzKa3aDmpZg5HgDHQqQ_lgl9L4rDWiWk7xd1igfAE,depth=10,body=Historical 82 (batch=0),prevs=['$8Ov9djlzX3ixwCWaULf6-SJNBuV8eeslfBVvt5b1Uds']", "event_id=$8Ov9djlzX3ixwCWaULf6-SJNBuV8eeslfBVvt5b1Uds,depth=10,body=Historical 81 (batch=0),prevs=['$Xflk7Gara-ISOAblxE3Vh5qbRDxJal8NLWSD6BC3wfY']", "event_id=$Xflk7Gara-ISOAblxE3Vh5qbRDxJal8NLWSD6BC3wfY,depth=10,body=Historical 80 (batch=0),prevs=['$cd4vq5xB0kqHa7HziMmI1U5njSwDr6NOz0qkvCrUeiA']", "event_id=$cd4vq5xB0kqHa7HziMmI1U5njSwDr6NOz0qkvCrUeiA,depth=10,body=Historical 79 (batch=0),prevs=['$xblwBKeHnnXlcK9nusq0mWw712br9ySr3elpt5vqYG8']", "event_id=$xblwBKeHnnXlcK9nusq0mWw712br9ySr3elpt5vqYG8,depth=10,body=Historical 78 (batch=0),prevs=['$cAnav2OUp8QsmQRbPc-PcX6A4rRW9UUsGFEGb1lgSyI']", "event_id=$cAnav2OUp8QsmQRbPc-PcX6A4rRW9UUsGFEGb1lgSyI,depth=10,body=Historical 77 (batch=0),prevs=['$3janYqE-QmrenClP9K2jxr-TM2AvG5H-kcgkM6F9UgA']", "event_id=$3janYqE-QmrenClP9K2jxr-TM2AvG5H-kcgkM6F9UgA,depth=10,body=Historical 76 (batch=0),prevs=['$UqMZVF90ZR0kzwt5q2noHEGCS8UFp_IAN4EbiMYVl4o']", "event_id=$UqMZVF90ZR0kzwt5q2noHEGCS8UFp_IAN4EbiMYVl4o,depth=10,body=Historical 75 (batch=0),prevs=['$D6IL83wLN4nkNwwikin8olaPEZCpzUh7-1PAOdD7HSY']", "event_id=$D6IL83wLN4nkNwwikin8olaPEZCpzUh7-1PAOdD7HSY,depth=10,body=Historical 74 (batch=0),prevs=['$MWVNPcLeQ_INUKlCdoOaySGlGeP5WwDt0myistqnTVk']", "event_id=$MWVNPcLeQ_INUKlCdoOaySGlGeP5WwDt0myistqnTVk,depth=10,body=Historical 73 (batch=0),prevs=['$Co1eNgILBaxBR_Gt7Vz_tIs0LutTc9CPvtYRyElNMj8']", "event_id=$Co1eNgILBaxBR_Gt7Vz_tIs0LutTc9CPvtYRyElNMj8,depth=10,body=Historical 72 (batch=0),prevs=['$ahJyDyEY1Oo8OAkrMZAeYDeTWNzx0B36GsQxcZWchFE']", "event_id=$ahJyDyEY1Oo8OAkrMZAeYDeTWNzx0B36GsQxcZWchFE,depth=10,body=Historical 71 (batch=0),prevs=['$JfGy5KmG-3KFMJFfcHsci6YvhvLq9pWok6GgcZlaXX8']", "event_id=$JfGy5KmG-3KFMJFfcHsci6YvhvLq9pWok6GgcZlaXX8,depth=10,body=Historical 70 (batch=0),prevs=['$VEfiWOEPPyCpN9z305k4IJySpEj5tl4InJb5mhFs7EA']", "event_id=$VEfiWOEPPyCpN9z305k4IJySpEj5tl4InJb5mhFs7EA,depth=10,body=Historical 69 (batch=0),prevs=['$FZswJ09DFvJSzGDFiA3JIdLuzFnG_TfObeEOES1fVTA']", "event_id=$FZswJ09DFvJSzGDFiA3JIdLuzFnG_TfObeEOES1fVTA,depth=10,body=Historical 68 (batch=0),prevs=['$fnzsLJ8duVIJ_S74QIozUaEwOp4G7j7fN5yRV-bzDHQ']", "event_id=$fnzsLJ8duVIJ_S74QIozUaEwOp4G7j7fN5yRV-bzDHQ,depth=10,body=Historical 67 (batch=0),prevs=['$vBWTNsLkJHP9bK0L1Wx3HdND6Rh8HhPmxBtXP7klwvE']", "event_id=$vBWTNsLkJHP9bK0L1Wx3HdND6Rh8HhPmxBtXP7klwvE,depth=10,body=Historical 66 (batch=0),prevs=['$y5PGi9vY9UcTEwTzeE18JB6-bjhf1GVeSEOph4ko-SE']", "event_id=$y5PGi9vY9UcTEwTzeE18JB6-bjhf1GVeSEOph4ko-SE,depth=10,body=Historical 65 (batch=0),prevs=['$7pM8GWYvhzxZ3zGkc76RLtsw7w02IT-sAiV8JkKeQCU']", "event_id=$7pM8GWYvhzxZ3zGkc76RLtsw7w02IT-sAiV8JkKeQCU,depth=10,body=Historical 64 (batch=0),prevs=['$wKaeK8hHhwLb1zHZaRJFE1C-n2kOtWnf076sNeWqaMc']", "event_id=$wKaeK8hHhwLb1zHZaRJFE1C-n2kOtWnf076sNeWqaMc,depth=10,body=Historical 63 (batch=0),prevs=['$8OHC-bni0whbdGfis5cfWL3--JACd0fx-pUBs5LuBXM']", "event_id=$8OHC-bni0whbdGfis5cfWL3--JACd0fx-pUBs5LuBXM,depth=10,body=Historical 62 (batch=0),prevs=['$nZ1WO04o85g1lkLQYlb72bZfx195YwuA-mQ3SsLEsuI']", "event_id=$nZ1WO04o85g1lkLQYlb72bZfx195YwuA-mQ3SsLEsuI,depth=10,body=Historical 61 (batch=0),prevs=['$rdA6ksM5YpvBrm9vHREi-d7x5AJCPZzrAAqxNN5I2ms']", "event_id=$rdA6ksM5YpvBrm9vHREi-d7x5AJCPZzrAAqxNN5I2ms,depth=10,body=Historical 60 (batch=0),prevs=['$v-3prgvdtmX8ySVzXby4Lw6zrncPxKdiLrDnM94y3K4']", "event_id=$v-3prgvdtmX8ySVzXby4Lw6zrncPxKdiLrDnM94y3K4,depth=10,body=Historical 59 (batch=0),prevs=['$ahhYJXd-_gSSZ_mROiEvBI37nYgYCi4WsrvG8sfEQ6g']", "event_id=$ahhYJXd-_gSSZ_mROiEvBI37nYgYCi4WsrvG8sfEQ6g,depth=10,body=Historical 58 (batch=0),prevs=['$JHFT5ugpxzbU6MmFJZ3wXN0hK33EX3-9As0MemxjsX0']", "event_id=$JHFT5ugpxzbU6MmFJZ3wXN0hK33EX3-9As0MemxjsX0,depth=10,body=Historical 57 (batch=0),prevs=['$JYLcmedG2JdmtBQnnXbIEI34QyeaJrMGLAJt_Mi_vuY']", "event_id=$JYLcmedG2JdmtBQnnXbIEI34QyeaJrMGLAJt_Mi_vuY,depth=10,body=Historical 56 (batch=0),prevs=['$nFeaEcPilAYFoXcid_5mz5jDEqIyvTZLYX4erRf2jlM']", "event_id=$nFeaEcPilAYFoXcid_5mz5jDEqIyvTZLYX4erRf2jlM,depth=10,body=Historical 55 (batch=0),prevs=['$ig5cEWTkkNG-ox84UUjG35G6WEHu-qpqAwYKQSOwHxQ']", "event_id=$ig5cEWTkkNG-ox84UUjG35G6WEHu-qpqAwYKQSOwHxQ,depth=10,body=Historical 54 (batch=0),prevs=['$2MDm0b5ztPCyLlAThSBT38Ie3F0ZG3sAE0r3to3yhtc']", "event_id=$2MDm0b5ztPCyLlAThSBT38Ie3F0ZG3sAE0r3to3yhtc,depth=10,body=Historical 53 (batch=0),prevs=['$gZ_e1ws9bh_uKzY4UskhVCO4IFikMi-waYooJbSJAOI']", "event_id=$gZ_e1ws9bh_uKzY4UskhVCO4IFikMi-waYooJbSJAOI,depth=10,body=Historical 52 (batch=0),prevs=['$pOBdCutp83GhxQkP9_tCyYeaN7YvEo0fuf5MiRsu_ew']", "event_id=$pOBdCutp83GhxQkP9_tCyYeaN7YvEo0fuf5MiRsu_ew,depth=10,body=Historical 51 (batch=0),prevs=['$HCC02P42Zfcn-vYaZrhQu1IdHOMVQrsc6XsfIGPrWNE']", "event_id=$HCC02P42Zfcn-vYaZrhQu1IdHOMVQrsc6XsfIGPrWNE,depth=10,body=Historical 50 (batch=0),prevs=['$R4sIu0EhjycD9c7qHb1Sfdw34PqzIFOiTzCsPs_Z-Fk']", "event_id=$R4sIu0EhjycD9c7qHb1Sfdw34PqzIFOiTzCsPs_Z-Fk,depth=10,body=Historical 49 (batch=0),prevs=['$Gdo7IK1ZUK3pHvHvOETByF8J_PLqVP6BDk1qI_3AZ90']", "event_id=$Gdo7IK1ZUK3pHvHvOETByF8J_PLqVP6BDk1qI_3AZ90,depth=10,body=Historical 48 (batch=0),prevs=['$VJ5qpulzoG4vkhkWnbwpgtBMHAMQSmhUB76lrI_2KXc']", "event_id=$VJ5qpulzoG4vkhkWnbwpgtBMHAMQSmhUB76lrI_2KXc,depth=10,body=Historical 47 (batch=0),prevs=['$-h0DqfdRxhEi6RMiaoKEHqzRDqA5Qy8ZskPGAp5rUI0']", "event_id=$-h0DqfdRxhEi6RMiaoKEHqzRDqA5Qy8ZskPGAp5rUI0,depth=10,body=Historical 46 (batch=0),prevs=['$_L-oB9rLJsXOnJkjnUQNDUL1C3jw2-YNbTBDiKG958Q']", "event_id=$_L-oB9rLJsXOnJkjnUQNDUL1C3jw2-YNbTBDiKG958Q,depth=10,body=Historical 45 (batch=0),prevs=['$iQeszxPn42bPGUQknk5TCYIWxVqnGDG-h4Bk1JFH_-4']", "event_id=$iQeszxPn42bPGUQknk5TCYIWxVqnGDG-h4Bk1JFH_-4,depth=10,body=Historical 44 (batch=0),prevs=['$raFQyAel7PT5FUymMwF3SIOXxuMpX7aJyRFDfsyFTpo']", "event_id=$raFQyAel7PT5FUymMwF3SIOXxuMpX7aJyRFDfsyFTpo,depth=10,body=Historical 43 (batch=0),prevs=['$bYfZTSvNiI6nOE7NvnwsaJBlDt8eGifjNWiBoOd-4JQ']", "event_id=$bYfZTSvNiI6nOE7NvnwsaJBlDt8eGifjNWiBoOd-4JQ,depth=10,body=Historical 42 (batch=0),prevs=['$iflmqGrv2ERr58y8QWMW8cAYjIwctoOqc_S8Zpxf2LE']", "event_id=$iflmqGrv2ERr58y8QWMW8cAYjIwctoOqc_S8Zpxf2LE,depth=10,body=Historical 41 (batch=0),prevs=['$84bhYzG__UwK-Rc1T4UGKzQUcLGOTMB9cWikG6Np_hM']", "event_id=$84bhYzG__UwK-Rc1T4UGKzQUcLGOTMB9cWikG6Np_hM,depth=10,body=Historical 40 (batch=0),prevs=['$xqS8HMfkjk_9j2aHTisNv8oeocOq38nxxU8MrepqgMU']", "event_id=$xqS8HMfkjk_9j2aHTisNv8oeocOq38nxxU8MrepqgMU,depth=10,body=Historical 39 (batch=0),prevs=['$EChQxibb9iOk8GqNnZRL3dQ0EfcXJpbl942hiEf1uxo']", "event_id=$EChQxibb9iOk8GqNnZRL3dQ0EfcXJpbl942hiEf1uxo,depth=10,body=Historical 38 (batch=0),prevs=['$bzBMsp8mhPIhbhv8RBjSP8zwvJ6ix2HYoEeourkeRJE']", "event_id=$bzBMsp8mhPIhbhv8RBjSP8zwvJ6ix2HYoEeourkeRJE,depth=10,body=Historical 37 (batch=0),prevs=['$lDNy6ds20jjt077dA_f9nFJrAu6reMkA-a9cZaW6cpI']", "event_id=$lDNy6ds20jjt077dA_f9nFJrAu6reMkA-a9cZaW6cpI,depth=10,body=Historical 36 (batch=0),prevs=['$yMQY-EQTPizmc_vOIsWIcY-Acp7dBNJew27NlfDHduo']", "event_id=$yMQY-EQTPizmc_vOIsWIcY-Acp7dBNJew27NlfDHduo,depth=10,body=Historical 35 (batch=0),prevs=['$Y9NaQWbjDCwqc8TxXbCduEp2FWGNDwUA_ElYWF148T8']", "event_id=$Y9NaQWbjDCwqc8TxXbCduEp2FWGNDwUA_ElYWF148T8,depth=10,body=Historical 34 (batch=0),prevs=['$_if1SSgNjDIT1YefAeoQ0z2aH6pe2jynJUDraBZOCjc']", "event_id=$_if1SSgNjDIT1YefAeoQ0z2aH6pe2jynJUDraBZOCjc,depth=10,body=Historical 33 (batch=0),prevs=['$_JgwcI13PE8e2MuebXSLBIyd_MaIorlEZtF_z_uIVHo']", "event_id=$_JgwcI13PE8e2MuebXSLBIyd_MaIorlEZtF_z_uIVHo,depth=10,body=Historical 32 (batch=0),prevs=['$CRZ4J3otfULJ7Iw1MaAQvIeakECsoZQs1Ehh-xeaC_k']", "event_id=$CRZ4J3otfULJ7Iw1MaAQvIeakECsoZQs1Ehh-xeaC_k,depth=10,body=Historical 31 (batch=0),prevs=['$qaIW_OYBoW7I03J5DnXMEJkRMKtnOOO5ttQjniYzHnI']", "event_id=$qaIW_OYBoW7I03J5DnXMEJkRMKtnOOO5ttQjniYzHnI,depth=10,body=Historical 30 (batch=0),prevs=['$9VNFmDvQZiJo50kQf1qPTAQyCPsdHjx23qC0DT34wY0']", "event_id=$9VNFmDvQZiJo50kQf1qPTAQyCPsdHjx23qC0DT34wY0,depth=10,body=Historical 29 (batch=0),prevs=['$gFbNkb60KtE_SzR9oi16tpkKE9k4hijC2A6dt66Al4k']", "event_id=$gFbNkb60KtE_SzR9oi16tpkKE9k4hijC2A6dt66Al4k,depth=10,body=Historical 28 (batch=0),prevs=['$Mq-dMSTxKEwHS3gPvYwU-Obqs3-Tp3SWjRPUq2VDvJg']", "event_id=$Mq-dMSTxKEwHS3gPvYwU-Obqs3-Tp3SWjRPUq2VDvJg,depth=10,body=Historical 27 (batch=0),prevs=['$5EC7XJFkyMLzhujTLI_m9UXNEnn-4NKZj-DCouBgyt8']", "event_id=$5EC7XJFkyMLzhujTLI_m9UXNEnn-4NKZj-DCouBgyt8,depth=10,body=Historical 26 (batch=0),prevs=['$UxlPuejLS1jHkUNCMRe2vNmc2tcuut-G2N8q6RFTfVQ']", "event_id=$UxlPuejLS1jHkUNCMRe2vNmc2tcuut-G2N8q6RFTfVQ,depth=10,body=Historical 25 (batch=0),prevs=['$BNqTjoMam0rgznru1lqk1U8cCqlJRA4GwORVLBeFYTU']", "event_id=$BNqTjoMam0rgznru1lqk1U8cCqlJRA4GwORVLBeFYTU,depth=10,body=Historical 24 (batch=0),prevs=['$7SdwqRy6BHBnNb_sQtj2Rph9era4F68ZM6mGPzjJthA']", "event_id=$7SdwqRy6BHBnNb_sQtj2Rph9era4F68ZM6mGPzjJthA,depth=10,body=Historical 23 (batch=0),prevs=['$HiveBkhuxr5NIgx9wGhfrmttnGOi0VpeesmqtjpkqE4']", "event_id=$HiveBkhuxr5NIgx9wGhfrmttnGOi0VpeesmqtjpkqE4,depth=10,body=Historical 22 (batch=0),prevs=['$m7iJp9s_Stodzk1I1Lf8egDLbIdyvgJPjAxomFguHb4']", "event_id=$m7iJp9s_Stodzk1I1Lf8egDLbIdyvgJPjAxomFguHb4,depth=10,body=Historical 21 (batch=0),prevs=['$KRSKtmQJHl80Hn7UR3emVczq4-QgKycVop3lkSeppIs']", "event_id=$KRSKtmQJHl80Hn7UR3emVczq4-QgKycVop3lkSeppIs,depth=10,body=Historical 20 (batch=0),prevs=['$ZCZKQrCeoKEC6oKqwjhK_8EQqwq75b7fEop-GgYCTiU']", "event_id=$ZCZKQrCeoKEC6oKqwjhK_8EQqwq75b7fEop-GgYCTiU,depth=10,body=Historical 19 (batch=0),prevs=['$9pZEcOtzeGWbnmRS_zAQAhdCIE8DZ6_cJAR0tm2AG1k']", "event_id=$9pZEcOtzeGWbnmRS_zAQAhdCIE8DZ6_cJAR0tm2AG1k,depth=10,body=Historical 18 (batch=0),prevs=['$FFXzuSEKR53_PpJ6EBGp8FEBi5Aig2MIq-F1opufK10']", "event_id=$FFXzuSEKR53_PpJ6EBGp8FEBi5Aig2MIq-F1opufK10,depth=10,body=Historical 17 (batch=0),prevs=['$hr0MEvZerIeshwxLzAGbqMXQGOXU_u9H1wUPtzqa6gY']", "event_id=$hr0MEvZerIeshwxLzAGbqMXQGOXU_u9H1wUPtzqa6gY,depth=10,body=Historical 16 (batch=0),prevs=['$hjP2ryNx-yqznW3EKrBZE1DaGSbvK8lB_3cWUDDGTVk']", "event_id=$hjP2ryNx-yqznW3EKrBZE1DaGSbvK8lB_3cWUDDGTVk,depth=10,body=Historical 15 (batch=0),prevs=['$E24xBJItDTqteFc11s2kk2JuvpQjZ2fd-vEWJ7QS784']", "event_id=$E24xBJItDTqteFc11s2kk2JuvpQjZ2fd-vEWJ7QS784,depth=10,body=Historical 14 (batch=0),prevs=['$eOuhBaOKitgobbzyU5XVieMLzgoCITj9eG-ewSMDB9I']", "event_id=$eOuhBaOKitgobbzyU5XVieMLzgoCITj9eG-ewSMDB9I,depth=10,body=Historical 13 (batch=0),prevs=['$4F4Xhu7QKpLM-n_aFmCLrKRP6zw7lrDLAYFTPKn8jYo']", "event_id=$4F4Xhu7QKpLM-n_aFmCLrKRP6zw7lrDLAYFTPKn8jYo,depth=10,body=Historical 12 (batch=0),prevs=['$e0wqSykOpA8Y0yipT9643SVVQ3DMOnW11yxvqAj9JrE']", "event_id=$e0wqSykOpA8Y0yipT9643SVVQ3DMOnW11yxvqAj9JrE,depth=10,body=Historical 11 (batch=0),prevs=['$u5gxt1eo59odS1XqVS-kV2GcaitrzorppcqVj3scevI']", "event_id=$u5gxt1eo59odS1XqVS-kV2GcaitrzorppcqVj3scevI,depth=10,body=Historical 10 (batch=0),prevs=['$Z1E79_z_FcAnw4G9NOvx3WhyL2ACyYPgy6zLhomKVks']", "event_id=$Z1E79_z_FcAnw4G9NOvx3WhyL2ACyYPgy6zLhomKVks,depth=10,body=Historical 9 (batch=0),prevs=['$USyjgdL7fwX4BvmblVs2cicGO3WDRB-ox56298yZmAA']", "event_id=$USyjgdL7fwX4BvmblVs2cicGO3WDRB-ox56298yZmAA,depth=10,body=Historical 8 (batch=0),prevs=['$tggu2uQEL2BWVJiwqsqwCJUca3SEFeylCAbPw5DrIdo']", "event_id=$tggu2uQEL2BWVJiwqsqwCJUca3SEFeylCAbPw5DrIdo,depth=10,body=Historical 7 (batch=0),prevs=['$M0iINx5TifxVTJ6pz-lio7xIeSla1RIV_N9WIK5sf4c']", "event_id=$M0iINx5TifxVTJ6pz-lio7xIeSla1RIV_N9WIK5sf4c,depth=10,body=Historical 6 (batch=0),prevs=['$hEs6MN2xyjXqHRcWV53_23NeNm--al9b_5BnMYYDPMY']", "event_id=$hEs6MN2xyjXqHRcWV53_23NeNm--al9b_5BnMYYDPMY,depth=10,body=Historical 5 (batch=0),prevs=['$L0BJu5ZOx2l8AMivXMBCqrvymVu_nCQM4ZO23uv5tBU']", "event_id=$L0BJu5ZOx2l8AMivXMBCqrvymVu_nCQM4ZO23uv5tBU,depth=10,body=Historical 4 (batch=0),prevs=['$V5BZU5LpCbW6XvPLGls0QGRbo98uuf1KpMZ0fg8k_-o']"] new implementation backfill events(100)=[ "event_id=$V5BZU5LpCbW6XvPLGls0QGRbo98uuf1KpMZ0fg8k_-o,depth=10,body=Historical 3 (batch=0),prevs=['$lF2XlCU-QmhC0YF8Wn8Q-EfQXqDxF_usgPIZ28rNhII']", "event_id=$lF2XlCU-QmhC0YF8Wn8Q-EfQXqDxF_usgPIZ28rNhII,depth=10,body=Historical 2 (batch=0),prevs=['$Ghnwsm4CqaWopAMGjKm8piFuTEqSaWgl1X7esubZOco']", "event_id=$Ghnwsm4CqaWopAMGjKm8piFuTEqSaWgl1X7esubZOco,depth=10,body=Historical 1 (batch=0),prevs=['$WRV7zPTr4KknEcYUvPlfxMdxcmlrQSsXAfTTlSBpjko']", "event_id=$WRV7zPTr4KknEcYUvPlfxMdxcmlrQSsXAfTTlSBpjko,depth=10,body=Historical 0 (batch=0),prevs=['$30VmrbC0AauqzWQ9F2g_KTPUdJhH_r79olOmfZYe0Pg']", "event_id=$30VmrbC0AauqzWQ9F2g_KTPUdJhH_r79olOmfZYe0Pg,depth=10,body=org.matrix.msc2716.insertion,prevs=['$an2lLzRXApWnCuRNrb1jTfMY9lWbz909AxyIRWy5-C0']", "event_id=$CmUx1RcXqn9QwVSKFKGmwVXwEFE9sHpnac5xvCgm-Co,depth=10,body=org.matrix.msc2716.batch,prevs=['$Kt2_lR2GlI6YoDY7q1ttaoIUv_s8WsjKgce6PT8utIA']", "event_id=$Kt2_lR2GlI6YoDY7q1ttaoIUv_s8WsjKgce6PT8utIA,depth=10,body=Historical 99 (batch=1),prevs=['$eaZav6ZipscONXQMtzaWQrTmiw9yo6_ifeaiI2VTJJM']", "event_id=$eaZav6ZipscONXQMtzaWQrTmiw9yo6_ifeaiI2VTJJM,depth=10,body=Historical 98 (batch=1),prevs=['$sSIhVeKMkHlwvQmCdUxouZyc0A7oBK_IcTYYPhQHniY']", "event_id=$sSIhVeKMkHlwvQmCdUxouZyc0A7oBK_IcTYYPhQHniY,depth=10,body=Historical 97 (batch=1),prevs=['$tQ6z1P58SRR3qqS6UnjWn85zP6CoU882ehdCkNcwhA8']", "event_id=$tQ6z1P58SRR3qqS6UnjWn85zP6CoU882ehdCkNcwhA8,depth=10,body=Historical 96 (batch=1),prevs=['$mh2o2cBnVjU5uQr0UAMPAWPpGsRzIKKpzc54Q2Z9JGY']", "event_id=$mh2o2cBnVjU5uQr0UAMPAWPpGsRzIKKpzc54Q2Z9JGY,depth=10,body=Historical 95 (batch=1),prevs=['$O_SPFvep-jjnHsW2484GzRYIDZkJhZYVgmCstDR3Qn0']", "event_id=$O_SPFvep-jjnHsW2484GzRYIDZkJhZYVgmCstDR3Qn0,depth=10,body=Historical 94 (batch=1),prevs=['$MkZU6zPnF5L4Mb9Y0AkOlWSG6S5yKhyVsFPm_WPc7EU']", "event_id=$MkZU6zPnF5L4Mb9Y0AkOlWSG6S5yKhyVsFPm_WPc7EU,depth=10,body=Historical 93 (batch=1),prevs=['$lts-3h-4-93qb12QQ8PwkvSu4bSJVKulvj_N2UPRhvs']", "event_id=$lts-3h-4-93qb12QQ8PwkvSu4bSJVKulvj_N2UPRhvs,depth=10,body=Historical 92 (batch=1),prevs=['$abLL8g1D70BE9EqtgSGoVjtbVK6QKG0KZhgBeJmEAuw']", "event_id=$abLL8g1D70BE9EqtgSGoVjtbVK6QKG0KZhgBeJmEAuw,depth=10,body=Historical 91 (batch=1),prevs=['$H2-ZSHXeJII1Jc3XvEjFfmjGUZgpsmlay7YuKKlF7JQ']", "event_id=$H2-ZSHXeJII1Jc3XvEjFfmjGUZgpsmlay7YuKKlF7JQ,depth=10,body=Historical 90 (batch=1),prevs=['$VsrkCzgyKUyPT4XpS0pa-K-CwRJ7MeNu9ictxOHPZo4']", "event_id=$VsrkCzgyKUyPT4XpS0pa-K-CwRJ7MeNu9ictxOHPZo4,depth=10,body=Historical 89 (batch=1),prevs=['$NSVKzqK5ucXWwWq7j4yfADj5m-8GqWFCWeqo2hvUDWg']", "event_id=$NSVKzqK5ucXWwWq7j4yfADj5m-8GqWFCWeqo2hvUDWg,depth=10,body=Historical 88 (batch=1),prevs=['$y5lPUOYBw0g12eIgDUdTn5QnmXD2QeFks61Ivpha1XA']", "event_id=$y5lPUOYBw0g12eIgDUdTn5QnmXD2QeFks61Ivpha1XA,depth=10,body=Historical 87 (batch=1),prevs=['$BdehWtfVHIxjlD3-pWh-6kcb1w2oPhAhgCWxNUkKAD8']", "event_id=$BdehWtfVHIxjlD3-pWh-6kcb1w2oPhAhgCWxNUkKAD8,depth=10,body=Historical 86 (batch=1),prevs=['$B0afYanjEywl1q1fYru7nL2RmjjZpF0gjTVQ7n9fY6k']", "event_id=$B0afYanjEywl1q1fYru7nL2RmjjZpF0gjTVQ7n9fY6k,depth=10,body=Historical 85 (batch=1),prevs=['$F2cOk9oSDKNSlrMoJioWtLi8AdhMkS-QpVwZw2qhWgM']", "event_id=$F2cOk9oSDKNSlrMoJioWtLi8AdhMkS-QpVwZw2qhWgM,depth=10,body=Historical 84 (batch=1),prevs=['$QGCvZWFsgW4KTskjaSqxjTenOa3TbbpBDWVyXESXDGI']", "event_id=$QGCvZWFsgW4KTskjaSqxjTenOa3TbbpBDWVyXESXDGI,depth=10,body=Historical 83 (batch=1),prevs=['$5evMhA_hLx02z7ydMm_cRfSUsC9OX0MJLayMydLrzN0']", "event_id=$5evMhA_hLx02z7ydMm_cRfSUsC9OX0MJLayMydLrzN0,depth=10,body=Historical 82 (batch=1),prevs=['$4jmsKkx-6NRVMCflN_s6sOfC053Zw1wOfCCM0bWlIX8']", "event_id=$4jmsKkx-6NRVMCflN_s6sOfC053Zw1wOfCCM0bWlIX8,depth=10,body=Historical 81 (batch=1),prevs=['$sTtp0fKA9BG4GDkJ8CUBQVNqpDDcf_a-PkGhJkGqzrw']", "event_id=$sTtp0fKA9BG4GDkJ8CUBQVNqpDDcf_a-PkGhJkGqzrw,depth=10,body=Historical 80 (batch=1),prevs=['$96go7TqZzPq75eOOLiDY0FkylHrmAGly0MTwX4_McMs']", "event_id=$96go7TqZzPq75eOOLiDY0FkylHrmAGly0MTwX4_McMs,depth=10,body=Historical 79 (batch=1),prevs=['$kUiJWkPHn1nPrUDzZJvWhTz1T8UBVELMlEdJGOEB_Qo']", "event_id=$kUiJWkPHn1nPrUDzZJvWhTz1T8UBVELMlEdJGOEB_Qo,depth=10,body=Historical 78 (batch=1),prevs=['$5GCJZEmchNvy0ikP7K7ZlvFb9FS6aVKRedx-08ZAF3s']", "event_id=$5GCJZEmchNvy0ikP7K7ZlvFb9FS6aVKRedx-08ZAF3s,depth=10,body=Historical 77 (batch=1),prevs=['$A3r2LkTKaHxBQh1Ailt4dCtaESyo53Ss1u1s3QiE_k0']", "event_id=$A3r2LkTKaHxBQh1Ailt4dCtaESyo53Ss1u1s3QiE_k0,depth=10,body=Historical 76 (batch=1),prevs=['$R7RRyXo6Msc3-368vZpiG84DfqOAiKLiSbV0ECGv6Oo']", "event_id=$R7RRyXo6Msc3-368vZpiG84DfqOAiKLiSbV0ECGv6Oo,depth=10,body=Historical 75 (batch=1),prevs=['$jmkYlZN6-oiQOi-bJfs0whVS7kKeJJUN6YJQfubbaEU']", "event_id=$jmkYlZN6-oiQOi-bJfs0whVS7kKeJJUN6YJQfubbaEU,depth=10,body=Historical 74 (batch=1),prevs=['$IuGVv6TZj5ZoOP4HHqZsKPWQ9HDTHS4QV7oFHk4kl7o']", "event_id=$IuGVv6TZj5ZoOP4HHqZsKPWQ9HDTHS4QV7oFHk4kl7o,depth=10,body=Historical 73 (batch=1),prevs=['$sNds_pVGUFf41AJwHmA1Sy7MUX1N9MtHGfw9ye6EItM']", "event_id=$sNds_pVGUFf41AJwHmA1Sy7MUX1N9MtHGfw9ye6EItM,depth=10,body=Historical 72 (batch=1),prevs=['$u0R7zUNs4BpneteeM7PWgGZE4XzQJuAwUtFSQGwTNI0']", "event_id=$u0R7zUNs4BpneteeM7PWgGZE4XzQJuAwUtFSQGwTNI0,depth=10,body=Historical 71 (batch=1),prevs=['$VthGtqcIDd04US7pMNTr0gSmR7PIOPLmDOljx1qNleQ']", "event_id=$VthGtqcIDd04US7pMNTr0gSmR7PIOPLmDOljx1qNleQ,depth=10,body=Historical 70 (batch=1),prevs=['$qYlzglfNCN1Br-4UkajF4c-SKYI0By5gExUxczeDsg4']", "event_id=$qYlzglfNCN1Br-4UkajF4c-SKYI0By5gExUxczeDsg4,depth=10,body=Historical 69 (batch=1),prevs=['$BsxtZqcTIs1aqFRsXtapKXYF6w2GiUEswpl6kH6S_4o']", "event_id=$BsxtZqcTIs1aqFRsXtapKXYF6w2GiUEswpl6kH6S_4o,depth=10,body=Historical 68 (batch=1),prevs=['$PmOzgtCDJCyGowxB8gWcYbfA2KUX0MCJ3xTE48FhAyY']", "event_id=$PmOzgtCDJCyGowxB8gWcYbfA2KUX0MCJ3xTE48FhAyY,depth=10,body=Historical 67 (batch=1),prevs=['$vZq13XMjZRJdFefxBg0L1U9shZq1TbArtR43YkQG7i8']", "event_id=$vZq13XMjZRJdFefxBg0L1U9shZq1TbArtR43YkQG7i8,depth=10,body=Historical 66 (batch=1),prevs=['$526Rm38ribM0XL3HltTbjVCUa3cDUdBh0V-9AMPtsYY']", "event_id=$526Rm38ribM0XL3HltTbjVCUa3cDUdBh0V-9AMPtsYY,depth=10,body=Historical 65 (batch=1),prevs=['$-sPB5z3uyN5tr4KezPKt6xu_yoaqgSDKhkzTJ0Um7yI']", "event_id=$-sPB5z3uyN5tr4KezPKt6xu_yoaqgSDKhkzTJ0Um7yI,depth=10,body=Historical 64 (batch=1),prevs=['$y5mX5SlScEAyPbUx1UA2-_ml0abbh0nGFYgJzF0XzZ0']", "event_id=$y5mX5SlScEAyPbUx1UA2-_ml0abbh0nGFYgJzF0XzZ0,depth=10,body=Historical 63 (batch=1),prevs=['$erT1ogi6tp9tF33_N3ZBLpIMBXOunszhADMYjs8U5AE']", "event_id=$erT1ogi6tp9tF33_N3ZBLpIMBXOunszhADMYjs8U5AE,depth=10,body=Historical 62 (batch=1),prevs=['$d9qTo4Ray7GJWJPm-CHkfZmZC0WmD8-K6Z1Riyot3fw']", "event_id=$d9qTo4Ray7GJWJPm-CHkfZmZC0WmD8-K6Z1Riyot3fw,depth=10,body=Historical 61 (batch=1),prevs=['$TVgdscs2fWtvx0ak6V0ljabU_4cc6UH18Y1r3OvDLYc']", "event_id=$TVgdscs2fWtvx0ak6V0ljabU_4cc6UH18Y1r3OvDLYc,depth=10,body=Historical 60 (batch=1),prevs=['$eMi_3TfsgYw2Fs99AJpr_9KAapZq3uwtv2fUDQ2weTc']", "event_id=$eMi_3TfsgYw2Fs99AJpr_9KAapZq3uwtv2fUDQ2weTc,depth=10,body=Historical 59 (batch=1),prevs=['$UBL_4l45DMvVTE12gyRaSDMw_fi_biDx1dL_gWz_v-g']", "event_id=$UBL_4l45DMvVTE12gyRaSDMw_fi_biDx1dL_gWz_v-g,depth=10,body=Historical 58 (batch=1),prevs=['$A6-UMuau--ffdnOA0NqWQK9uO0NBCB_rwVCvu_hSAOw']", "event_id=$A6-UMuau--ffdnOA0NqWQK9uO0NBCB_rwVCvu_hSAOw,depth=10,body=Historical 57 (batch=1),prevs=['$Eia3O1ptxGtKhINTEVX_FxBU3TjL0MBp1XdeFCStcCg']", "event_id=$Eia3O1ptxGtKhINTEVX_FxBU3TjL0MBp1XdeFCStcCg,depth=10,body=Historical 56 (batch=1),prevs=['$Pb5OyRAHNrXDyPt8F1dTWVGi-Zvqkf-HqzCwcu0salY']", "event_id=$Pb5OyRAHNrXDyPt8F1dTWVGi-Zvqkf-HqzCwcu0salY,depth=10,body=Historical 55 (batch=1),prevs=['$c1KPN_U0VF8G3jGnD6UpQob_h9X_KQfXv4lciJUb4PQ']", "event_id=$c1KPN_U0VF8G3jGnD6UpQob_h9X_KQfXv4lciJUb4PQ,depth=10,body=Historical 54 (batch=1),prevs=['$fwWir53iSGaGEMRt-mQe6hYmkwIQGjDojFO08zlzgX4']", "event_id=$fwWir53iSGaGEMRt-mQe6hYmkwIQGjDojFO08zlzgX4,depth=10,body=Historical 53 (batch=1),prevs=['$TmacWSD6J8y8vERZfA-IdRpDeMS_4A1HFNUBg_W5MaA']", "event_id=$TmacWSD6J8y8vERZfA-IdRpDeMS_4A1HFNUBg_W5MaA,depth=10,body=Historical 52 (batch=1),prevs=['$_xBvb_mIUJgyvA-rgBppyjXVZ3SXn03isQ0axMOtThc']", "event_id=$_xBvb_mIUJgyvA-rgBppyjXVZ3SXn03isQ0axMOtThc,depth=10,body=Historical 51 (batch=1),prevs=['$AYD-cd4lrJLbW35mCK8dKZEZI705gOsA1iiyW3uqDWk']", "event_id=$AYD-cd4lrJLbW35mCK8dKZEZI705gOsA1iiyW3uqDWk,depth=10,body=Historical 50 (batch=1),prevs=['$KK1BXAZ3E-U7qKWhO6Y6DfXedQ04BhwuA_bNI5Nee8g']", "event_id=$KK1BXAZ3E-U7qKWhO6Y6DfXedQ04BhwuA_bNI5Nee8g,depth=10,body=Historical 49 (batch=1),prevs=['$_oLPTYdg92q_l3c6vxV4tO9ioR9nPrk1lyb2qiLIPOk']", "event_id=$_oLPTYdg92q_l3c6vxV4tO9ioR9nPrk1lyb2qiLIPOk,depth=10,body=Historical 48 (batch=1),prevs=['$RZ0Mzvftn99rU6XFl32jKHs0R6YYaMPy_HJy41tK4ig']", "event_id=$RZ0Mzvftn99rU6XFl32jKHs0R6YYaMPy_HJy41tK4ig,depth=10,body=Historical 47 (batch=1),prevs=['$mloZAG1tKqfjIBQhJk8Mn7M5ER84cJ7bC0TW3rOsKpU']", "event_id=$mloZAG1tKqfjIBQhJk8Mn7M5ER84cJ7bC0TW3rOsKpU,depth=10,body=Historical 46 (batch=1),prevs=['$RY6YWeTkTc8aQP6ZkPq_vL0qs1kg53QeDcKmrLi0ASw']", "event_id=$RY6YWeTkTc8aQP6ZkPq_vL0qs1kg53QeDcKmrLi0ASw,depth=10,body=Historical 45 (batch=1),prevs=['$5A2e6qJj9n9ejvUW6TcVkm6-cuHghnRKQDW3T3a4eqI']", "event_id=$5A2e6qJj9n9ejvUW6TcVkm6-cuHghnRKQDW3T3a4eqI,depth=10,body=Historical 44 (batch=1),prevs=['$KDMWnrkM1ezjDXlSrXjvhZS-RRuJc6_hrX-Msh0gP7o']", "event_id=$KDMWnrkM1ezjDXlSrXjvhZS-RRuJc6_hrX-Msh0gP7o,depth=10,body=Historical 43 (batch=1),prevs=['$9Q6SSr_YMaQ6oc9w7lYs-bRw0cNESLPTE45Fg2_Zzv4']", "event_id=$9Q6SSr_YMaQ6oc9w7lYs-bRw0cNESLPTE45Fg2_Zzv4,depth=10,body=Historical 42 (batch=1),prevs=['$b2QYayqOOaCxQqs655Tve2OKeXrT9CJeVIchVUn00hI']", "event_id=$b2QYayqOOaCxQqs655Tve2OKeXrT9CJeVIchVUn00hI,depth=10,body=Historical 41 (batch=1),prevs=['$xQQBp4Dwm-3lxCWkJsck9bXuQoCKsdPIortpc79nTtw']", "event_id=$xQQBp4Dwm-3lxCWkJsck9bXuQoCKsdPIortpc79nTtw,depth=10,body=Historical 40 (batch=1),prevs=['$RgTwPhV6RzdVBmuqoYBNblHPZ-qpYuTCshDBrfpwvds']", "event_id=$RgTwPhV6RzdVBmuqoYBNblHPZ-qpYuTCshDBrfpwvds,depth=10,body=Historical 39 (batch=1),prevs=['$RWyv_RX_EoNzCDnHSCfx6fDnJrhWl46iWYomZSS2GsQ']", "event_id=$RWyv_RX_EoNzCDnHSCfx6fDnJrhWl46iWYomZSS2GsQ,depth=10,body=Historical 38 (batch=1),prevs=['$Sj78rQcr9DuCnhtcYuZjWS6o1ZiFDd3ZQxDtJKxjeNg']", "event_id=$Sj78rQcr9DuCnhtcYuZjWS6o1ZiFDd3ZQxDtJKxjeNg,depth=10,body=Historical 37 (batch=1),prevs=['$KePkqP3sJgnC8wVMyRFd0v7Fsu-8LiLTZbhx0JsWfnY']", "event_id=$KePkqP3sJgnC8wVMyRFd0v7Fsu-8LiLTZbhx0JsWfnY,depth=10,body=Historical 36 (batch=1),prevs=['$QdyPRKCNju-Rzm1u530j3N7AEcRFA5mAO3sVbBZunkQ']", "event_id=$QdyPRKCNju-Rzm1u530j3N7AEcRFA5mAO3sVbBZunkQ,depth=10,body=Historical 35 (batch=1),prevs=['$dd3HF7wo7YRhA6e0j5I9eGZtaH1rMK7dAriiNTQd7-Y']", "event_id=$dd3HF7wo7YRhA6e0j5I9eGZtaH1rMK7dAriiNTQd7-Y,depth=10,body=Historical 34 (batch=1),prevs=['$l6q9M7E_yWQID_v4YZMYZfNevVqhK0Pm6qIDABcx7ag']", "event_id=$l6q9M7E_yWQID_v4YZMYZfNevVqhK0Pm6qIDABcx7ag,depth=10,body=Historical 33 (batch=1),prevs=['$GAdAhlUMns9H8jeNH6yrfNp1c4_-lhQKyBnmAkQNZ5A']", "event_id=$GAdAhlUMns9H8jeNH6yrfNp1c4_-lhQKyBnmAkQNZ5A,depth=10,body=Historical 32 (batch=1),prevs=['$6liLB-IPWtTynCd27je2vCoHfDK_S3pwlRe5GoUsLY8']", "event_id=$6liLB-IPWtTynCd27je2vCoHfDK_S3pwlRe5GoUsLY8,depth=10,body=Historical 31 (batch=1),prevs=['$9C_ywoPrhy68x4-MurcodrKje7FOw5GuG9yMJ6W2ObM']", "event_id=$9C_ywoPrhy68x4-MurcodrKje7FOw5GuG9yMJ6W2ObM,depth=10,body=Historical 30 (batch=1),prevs=['$QyvDk3PQ4dEDxHb0ihFqAwWpqH-NQ86GV_w5njGNOSY']", "event_id=$QyvDk3PQ4dEDxHb0ihFqAwWpqH-NQ86GV_w5njGNOSY,depth=10,body=Historical 29 (batch=1),prevs=['$c2ee8wW8tAvev6UB1yOZMA56j-BFcxZJQqPbXxiqmeg']", "event_id=$c2ee8wW8tAvev6UB1yOZMA56j-BFcxZJQqPbXxiqmeg,depth=10,body=Historical 28 (batch=1),prevs=['$psirWB7VB7WUVg7IjiG1XX2WMOAiN_FFcdo8crQ3KF0']", "event_id=$psirWB7VB7WUVg7IjiG1XX2WMOAiN_FFcdo8crQ3KF0,depth=10,body=Historical 27 (batch=1),prevs=['$AMyblWA-Jny9Nobd-ghaFit6ZDG6OOg3rTxwEIn6zkQ']", "event_id=$AMyblWA-Jny9Nobd-ghaFit6ZDG6OOg3rTxwEIn6zkQ,depth=10,body=Historical 26 (batch=1),prevs=['$EtDL8yGxA03ZO5pTeztYOQvHxUaMCCz9Nj7oGgM0CL8']", "event_id=$EtDL8yGxA03ZO5pTeztYOQvHxUaMCCz9Nj7oGgM0CL8,depth=10,body=Historical 25 (batch=1),prevs=['$zmqDOTVLDS_khXDmPiPvJ7745wI_yuEAsxS_fyDroIo']", "event_id=$zmqDOTVLDS_khXDmPiPvJ7745wI_yuEAsxS_fyDroIo,depth=10,body=Historical 24 (batch=1),prevs=['$lMcNfcLZKCtmrGISzGo3M_lxJapImAmdky9Lezij6TI']", "event_id=$lMcNfcLZKCtmrGISzGo3M_lxJapImAmdky9Lezij6TI,depth=10,body=Historical 23 (batch=1),prevs=['$i4vpa73GXD7wZB2r2t-8nV--pGq_GdKCotM2-wpVJ10']", "event_id=$i4vpa73GXD7wZB2r2t-8nV--pGq_GdKCotM2-wpVJ10,depth=10,body=Historical 22 (batch=1),prevs=['$9BA_P9uGh-cLjLau5nToTmpsnxdaPkT2RjMf9ZYglNI']", "event_id=$9BA_P9uGh-cLjLau5nToTmpsnxdaPkT2RjMf9ZYglNI,depth=10,body=Historical 21 (batch=1),prevs=['$ySO1DPJTl_IiFwv4d6jbNy43a1go9rD_86A2EPIGVZE']", "event_id=$ySO1DPJTl_IiFwv4d6jbNy43a1go9rD_86A2EPIGVZE,depth=10,body=Historical 20 (batch=1),prevs=['$XOyDGiQEhlEoT8UGl4HIod5Zb29R3Op2BFIaVeiCRyY']", "event_id=$XOyDGiQEhlEoT8UGl4HIod5Zb29R3Op2BFIaVeiCRyY,depth=10,body=Historical 19 (batch=1),prevs=['$XDqJHr3kmi2Z9hpNX2Vz7o524cLEMkjKwk-_V2A3U1U']", "event_id=$XDqJHr3kmi2Z9hpNX2Vz7o524cLEMkjKwk-_V2A3U1U,depth=10,body=Historical 18 (batch=1),prevs=['$hlXaBZDCY4XSg-qjuzkAWhYqoBpdX8ehIFZd5M_9rRM']", "event_id=$hlXaBZDCY4XSg-qjuzkAWhYqoBpdX8ehIFZd5M_9rRM,depth=10,body=Historical 17 (batch=1),prevs=['$LMLomCl3U7zYXi4sIHHDnfznJkns0-DLRDGMbhJ1AAo']", "event_id=$LMLomCl3U7zYXi4sIHHDnfznJkns0-DLRDGMbhJ1AAo,depth=10,body=Historical 16 (batch=1),prevs=['$d0pjrKi7Ws1QM-XyB1_d9Yrg0AGY8MD6JLJqrnaaoJM']", "event_id=$d0pjrKi7Ws1QM-XyB1_d9Yrg0AGY8MD6JLJqrnaaoJM,depth=10,body=Historical 15 (batch=1),prevs=['$hx2dBPyOh6KnEzdpw7CnvoSOkiYTKJp82MXaWA_77N0']", "event_id=$hx2dBPyOh6KnEzdpw7CnvoSOkiYTKJp82MXaWA_77N0,depth=10,body=Historical 14 (batch=1),prevs=['$-y0R8tPCI4n4L0t2j4Ua3QIfau7n4kqPXgnQjb4e1qk']", "event_id=$-y0R8tPCI4n4L0t2j4Ua3QIfau7n4kqPXgnQjb4e1qk,depth=10,body=Historical 13 (batch=1),prevs=['$Af85_d8_UkHp8r8KzOnjCCI0whyMQB7U3G5u8NlMk4I']", "event_id=$Af85_d8_UkHp8r8KzOnjCCI0whyMQB7U3G5u8NlMk4I,depth=10,body=Historical 12 (batch=1),prevs=['$1nQJC9ULuCVGYWU_umE58wq5wHmmFx2GdgF6nmQt5VU']", "event_id=$1nQJC9ULuCVGYWU_umE58wq5wHmmFx2GdgF6nmQt5VU,depth=10,body=Historical 11 (batch=1),prevs=['$SdAp6h2pjv7aIxDSZJxwQbolYrqzUGf1gHOeBnqAULU']", "event_id=$SdAp6h2pjv7aIxDSZJxwQbolYrqzUGf1gHOeBnqAULU,depth=10,body=Historical 10 (batch=1),prevs=['$YDMkSKMoYDtwMd-nKvgAIk38uzc1UejIyBvXNoJ_RFM']", "event_id=$YDMkSKMoYDtwMd-nKvgAIk38uzc1UejIyBvXNoJ_RFM,depth=10,body=Historical 9 (batch=1),prevs=['$a8nvUo95cOf-cO4hcgfO1sT0kyWKq8XPUDmJe0DbxNE']", "event_id=$a8nvUo95cOf-cO4hcgfO1sT0kyWKq8XPUDmJe0DbxNE,depth=10,body=Historical 8 (batch=1),prevs=['$LunwOlIdMiDjrcQazQ6I3n0T8aLfmSA24fIEeth1hHs']", "event_id=$LunwOlIdMiDjrcQazQ6I3n0T8aLfmSA24fIEeth1hHs,depth=10,body=Historical 7 (batch=1),prevs=['$UOvHOp4SKtZqSDWZvD0s3VGdU23-6YqiWy_742cqY-8']", "event_id=$UOvHOp4SKtZqSDWZvD0s3VGdU23-6YqiWy_742cqY-8,depth=10,body=Historical 6 (batch=1),prevs=['$W-c9syEfTTufHnNBpuMUIk621ecs9W6bTeLW0OgWFfY']\n"] new implementation backfill events(17)=[ "event_id=$W-c9syEfTTufHnNBpuMUIk621ecs9W6bTeLW0OgWFfY,depth=10,body=Historical 5 (batch=1),prevs=['$Gq8-foARgSoEzcEwXhOsiey2pefpCX3vjnCLtsIdvU0']", "event_id=$Gq8-foARgSoEzcEwXhOsiey2pefpCX3vjnCLtsIdvU0,depth=10,body=Historical 4 (batch=1),prevs=['$toIu3hsoBDXkVJs91nUG0hodf3nuFkbidY7ztVPrtCM']", "event_id=$toIu3hsoBDXkVJs91nUG0hodf3nuFkbidY7ztVPrtCM,depth=10,body=Historical 3 (batch=1),prevs=['$SRcKQ-ggRujFzHbJC44rWvslb4YBKgieWahZakx3zMw']", "event_id=$SRcKQ-ggRujFzHbJC44rWvslb4YBKgieWahZakx3zMw,depth=10,body=Historical 2 (batch=1),prevs=['$Pgqvjn_rkQb6eEdQDeKclO-l_ODlnplF7S-rT1i_-64']", "event_id=$Pgqvjn_rkQb6eEdQDeKclO-l_ODlnplF7S-rT1i_-64,depth=10,body=Historical 1 (batch=1),prevs=['$jEW2Smmgh5tIh1QMtEqmluy-o3FrcVhNAH5e3R4jMoo']", "event_id=$jEW2Smmgh5tIh1QMtEqmluy-o3FrcVhNAH5e3R4jMoo,depth=10,body=Historical 0 (batch=1),prevs=['$1bbHCIrQLToHvum-PGxpIKhCmZ5MYLusAy_gJm7IWTw']", "event_id=$1bbHCIrQLToHvum-PGxpIKhCmZ5MYLusAy_gJm7IWTw,depth=10,body=org.matrix.msc2716.insertion,prevs=['$758TrMVR2MlE4EhpRUvCYZFcp11cbE27edQcwKaUSI8']", "event_id=$T5-OUXAW3CCYAHskqk5s0V2ZzJoQy5D8lQJ7LZFWvTI,depth=9,body=Message 1 (eventIDsBefore),prevs=['$WYEMee2J5PcH1yM2XDnD2XIhyMS_HHDdYXop6BOL69E']", "event_id=$WYEMee2J5PcH1yM2XDnD2XIhyMS_HHDdYXop6BOL69E,depth=8,body=Message 0 (eventIDsBefore),prevs=['$jTD4_ZLQQgXienBOsHMwSqyuuYtXQq4WYj7hnPbRC0o']", "event_id=$jTD4_ZLQQgXienBOsHMwSqyuuYtXQq4WYj7hnPbRC0o,depth=7,body=m.room.member,prevs=['$d9EI1fE4ArZLYeQPs3P6dlqTmtskwjJWtlfNEZTdSoA']", "event_id=$d9EI1fE4ArZLYeQPs3P6dlqTmtskwjJWtlfNEZTdSoA,depth=6,body=m.room.name,prevs=['$VvA6Hhp67rDLix7Pg9tZs6DDDh0gq_Mw-qr9wd5cO2k']", "event_id=$VvA6Hhp67rDLix7Pg9tZs6DDDh0gq_Mw-qr9wd5cO2k,depth=5,body=m.room.history_visibility,prevs=['$7fkTDT3EgbweX381_HInASE-jAzhxcQfkzOWuXxXO7U']", "event_id=$7fkTDT3EgbweX381_HInASE-jAzhxcQfkzOWuXxXO7U,depth=4,body=m.room.join_rules,prevs=['$uvO6_KB-CJtOtomqSgSAWLL59mJK0JW2U33ebFTuoK4']", "event_id=$uvO6_KB-CJtOtomqSgSAWLL59mJK0JW2U33ebFTuoK4,depth=3,body=m.room.power_levels,prevs=['$HjRQ8RpCPPWO4Ufq6bSCTqa_tSlKez15KvZRMpvvskc']", "event_id=$HjRQ8RpCPPWO4Ufq6bSCTqa_tSlKez15KvZRMpvvskc,depth=2,body=m.room.member,prevs=['$yXLeDjVhdX5jmjLVxmsxKhG_rjfA9ujL8Cd8eHNha6U']", 'event_id=$758TrMVR2MlE4EhpRUvCYZFcp11cbE27edQcwKaUSI8,depth=1,body=m.room.member,prevs=[]', 'event_id=$yXLeDjVhdX5jmjLVxmsxKhG_rjfA9ujL8Cd8eHNha6U,depth=1,body=m.room.create,prevs=[]' ] ```
-rw-r--r-- | synapse/federation/federation_client.py | 84 | ||||
-rw-r--r-- | synapse/handlers/federation.py | 184 |
2 files changed, 196 insertions, 72 deletions
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2ab4dec88f..eafffff2bc 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -264,6 +264,62 @@ class FederationClient(FederationBase): return pdus + async def get_pdu_from_destination_raw( + self, + destination: str, + event_id: str, + room_version: RoomVersion, + outlier: bool = False, + timeout: Optional[int] = None, + ) -> Optional[EventBase]: + """Requests the PDU with given origin and ID from the remote home + server. + + Does not have any caching or rate limiting! + + Args: + destination: Which homeserver to query + event_id: event to fetch + room_version: version of the room + outlier: Indicates whether the PDU is an `outlier`, i.e. if + it's from an arbitrary point in the context as opposed to part + of the current block of PDUs. Defaults to `False` + timeout: How long to try (in ms) each destination for before + moving to the next destination. None indicates no timeout. + + Returns: + The requested PDU, or None if we were unable to find it. + + Raises: + SynapseError, NotRetryingDestination, FederationDeniedError + """ + + signed_pdu = None + + transaction_data = await self.transport_layer.get_event( + destination, event_id, timeout=timeout + ) + + logger.info( + "retrieved event id %s from %s: %r", + event_id, + destination, + transaction_data, + ) + + pdu_list: List[EventBase] = [ + event_from_pdu_json(p, room_version, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list and pdu_list[0]: + pdu = pdu_list[0] + + # Check signatures are correct. + signed_pdu = await self._check_sigs_and_hash(room_version, pdu) + + return signed_pdu + async def get_pdu( self, destinations: Iterable[str], @@ -308,30 +364,14 @@ class FederationClient(FederationBase): continue try: - transaction_data = await self.transport_layer.get_event( - destination, event_id, timeout=timeout - ) - - logger.debug( - "retrieved event id %s from %s: %r", - event_id, - destination, - transaction_data, + signed_pdu = await self.get_pdu_from_destination_raw( + destination=destination, + event_id=event_id, + room_version=room_version, + outlier=outlier, + timeout=timeout, ) - pdu_list: List[EventBase] = [ - event_from_pdu_json(p, room_version, outlier=outlier) - for p in transaction_data["pdus"] - ] - - if pdu_list and pdu_list[0]: - pdu = pdu_list[0] - - # Check signatures are correct. - signed_pdu = await self._check_sigs_and_hash(room_version, pdu) - - break - pdu_attempts[destination] = now except SynapseError as e: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e28c74daf0..c2163c7e53 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -62,6 +62,7 @@ from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.retryutils import NotRetryingDestination from synapse.visibility import filter_events_for_server +from synapse.storage.databases.main.event_federation import BackfillQueueNavigationItem if TYPE_CHECKING: from synapse.server import HomeServer @@ -1043,8 +1044,16 @@ class FederationHandler: return [] async def get_backfill_events( - self, room_id: str, event_id_list: list, limit: int + self, origin: str, room_id: str, event_id_list: list, limit: int ) -> List[EventBase]: + logger.info( + "get_backfill_events(room_id=%s): seeding backfill with event_id_list=%s limit=%s origin=%s", + room_id, + event_id_list, + limit, + origin, + ) + event_id_results = set() # In a PriorityQueue, the lowest valued entries are retrieved first. @@ -1054,8 +1063,20 @@ class FederationHandler: # negative depth so that we process the newest-in-time messages first # going backwards in time. stream_ordering follows the same pattern. queue = PriorityQueue() - seed_events = await self.store.get_events_as_list(event_id_list) + logger.info( + "get_backfill_events(room_id=%s): seed_events=%s", + room_id, + [ + BackfillQueueNavigationItem( + depth=seed_event.depth, + stream_ordering=seed_event.internal_metadata.stream_ordering, + event_id=seed_event.event_id, + type=seed_event.type, + ) + for seed_event in seed_events + ], + ) for seed_event in seed_events: # Make sure the seed event actually pertains to this room. We also # need to make sure the depth is available since our whole DAG @@ -1079,8 +1100,7 @@ class FederationHandler: if event_id in event_id_results: continue - event_id_results.add(event_id) - + found_undiscovered_connected_historical_messages = False if self.hs.config.experimental.msc2716_enabled: # Try and find any potential historical batches of message history. # @@ -1093,8 +1113,10 @@ class FederationHandler: event_id, limit - len(event_id_results) ) ) - logger.debug( - "_get_backfill_events: connected_insertion_event_backfill_results=%s", + logger.info( + "get_backfill_events(room_id=%s): connected_insertion_event_backfill_results(%s)=%s", + room_id, + event_id, connected_insertion_event_backfill_results, ) for ( @@ -1104,15 +1126,63 @@ class FederationHandler: connected_insertion_event_backfill_item.event_id not in event_id_results ): - queue.put( - ( - -connected_insertion_event_backfill_item.depth, - -connected_insertion_event_backfill_item.stream_ordering, + # Check whether the insertion event is already on the + # federating homeserver we're trying to send backfill + # events to + room_version = await self.store.get_room_version(room_id) + event_exists_on_remote_server = None + try: + # Because of the nature of backfill giving events to + # the federated homeserver in one chunk and then we + # can possibly query about that same event in the + # next chunk, we need to avoid getting a cached + # response. We want to know *now* whether they have + # backfilled the insertion event. + event_exists_on_remote_server = await self.federation_client.get_pdu_from_destination_raw( + origin, connected_insertion_event_backfill_item.event_id, - connected_insertion_event_backfill_item.type, + room_version=room_version, + outlier=True, + timeout=10000, ) + except Exception as e: + logger.info( + "get_backfill_events(room_id=%s): Failed to fetch insertion event_id=%s from origin=%s but we're just going to assume it's not backfilled there yet. error=%s", + room_id, + connected_insertion_event_backfill_item.event_id, + origin, + e, + ) + + logger.info( + "get_backfill_events(room_id=%s): checked if insertion event_id=%s exists on federated homeserver(origin=%s) already? event_exists_on_remote_server=%s", + room_id, + connected_insertion_event_backfill_item.event_id, + origin, + event_exists_on_remote_server, ) + # If the event is already on the federated homeserver, + # we don't need to try to branch off onto this + # historical chain of messages. Below, we will instead + # just go up the `prev_events` as normal. + # + # This is important so that the first time we backfill + # the federated homeserver, we jump off and go down the + # historical branch. But after the historical branch is + # exhausted and the event comes up again in backfill, we + # will choose the "live" DAG. + if not event_exists_on_remote_server: + found_undiscovered_connected_historical_messages = True + queue.put( + ( + -connected_insertion_event_backfill_item.depth, + -connected_insertion_event_backfill_item.stream_ordering, + connected_insertion_event_backfill_item.event_id, + connected_insertion_event_backfill_item.type, + ) + ) + # Second, we need to go and try to find any batch events connected # to a given insertion event (by batch_id). If we find any, we'll # add them to the queue and navigate up the DAG like normal in the @@ -1123,8 +1193,10 @@ class FederationHandler: event_id, limit - len(event_id_results) ) ) - logger.debug( - "_get_backfill_events: connected_batch_event_backfill_results %s", + logger.info( + "get_backfill_events(room_id=%s): connected_batch_event_backfill_results(%s)=%s", + room_id, + event_id, connected_batch_event_backfill_results, ) for ( @@ -1143,28 +1215,39 @@ class FederationHandler: ) ) - # Now we just look up the DAG by prev_events as normal - connected_prev_event_backfill_results = ( - await self.store.get_connected_prev_event_backfill_results( - event_id, limit - len(event_id_results) + # If we found a historical branch of history off of the message lets + # navigate down that in the next iteration of the loop instead of + # the normal prev_event chain. + if not found_undiscovered_connected_historical_messages: + event_id_results.add(event_id) + + # Now we just look up the DAG by prev_events as normal + connected_prev_event_backfill_results = ( + await self.store.get_connected_prev_event_backfill_results( + event_id, limit - len(event_id_results) + ) ) - ) - logger.debug( - "_get_backfill_events: prev_event_ids %s", - connected_prev_event_backfill_results, - ) - for ( - connected_prev_event_backfill_item - ) in connected_prev_event_backfill_results: - if connected_prev_event_backfill_item.event_id not in event_id_results: - queue.put( - ( - -connected_prev_event_backfill_item.depth, - -connected_prev_event_backfill_item.stream_ordering, - connected_prev_event_backfill_item.event_id, - connected_prev_event_backfill_item.type, + logger.info( + "get_backfill_events(room_id=%s): connected_prev_event_backfill_results(%s)=%s", + room_id, + event_id, + connected_prev_event_backfill_results, + ) + for ( + connected_prev_event_backfill_item + ) in connected_prev_event_backfill_results: + if ( + connected_prev_event_backfill_item.event_id + not in event_id_results + ): + queue.put( + ( + -connected_prev_event_backfill_item.depth, + -connected_prev_event_backfill_item.stream_ordering, + connected_prev_event_backfill_item.event_id, + connected_prev_event_backfill_item.type, + ) ) - ) events = await self.store.get_events_as_list(event_id_results) return sorted( @@ -1182,24 +1265,25 @@ class FederationHandler: # Synapse asks for 100 events per backfill request. Do not allow more. limit = min(limit, 100) - events = await self.store.get_backfill_events(room_id, pdu_list, limit) - logger.info( - "old implementation backfill events=%s", - [ - "event_id=%s,depth=%d,body=%s,prevs=%s\n" - % ( - event.event_id, - event.depth, - event.content.get("body", event.type), - event.prev_event_ids(), - ) - for event in events - ], - ) - - events = await self.get_backfill_events(room_id, pdu_list, limit) + # events = await self.store.get_backfill_events(room_id, pdu_list, limit) + # logger.info( + # "old implementation backfill events=%s", + # [ + # "event_id=%s,depth=%d,body=%s,prevs=%s\n" + # % ( + # event.event_id, + # event.depth, + # event.content.get("body", event.type), + # event.prev_event_ids(), + # ) + # for event in events + # ], + # ) + + events = await self.get_backfill_events(origin, room_id, pdu_list, limit) logger.info( - "new implementation backfill events=%s", + "new implementation backfill events(%d)=%s", + len(events), [ "event_id=%s,depth=%d,body=%s,prevs=%s\n" % ( |