summary refs log tree commit diff
diff options
context:
space:
mode:
authorAdrian Tschira <nota@notafile.com>2018-04-29 13:54:38 +0200
committerAdrian Tschira <nota@notafile.com>2018-05-19 17:48:30 +0200
commit45b55e23d34371986ee0da5be784a1f2134fd58a (patch)
treef21e44820bdbca75ca1d976c7f937b980c7c3075
parentMerge pull request #3241 from matrix-org/fix_user_visits_insertion (diff)
downloadsynapse-45b55e23d34371986ee0da5be784a1f2134fd58a.tar.xz
Add batch_iter to utils
There's a frequent idiom I noticed where an iterable is split up into a
number of chunks/batches. Unfortunately that method does not work with
iterators like dict.keys() in python3. This implementation works with
iterators.

Signed-off-by: Adrian Tschira <nota@notafile.com>
-rw-r--r--synapse/util/__init__.py18
1 files changed, 18 insertions, 0 deletions
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 814a7bf71b..fc11e26623 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -20,6 +20,8 @@ from twisted.internet import defer, reactor, task
 import time
 import logging
 
+from itertools import islice
+
 logger = logging.getLogger(__name__)
 
 
@@ -79,3 +81,19 @@ class Clock(object):
         except Exception:
             if not ignore_errs:
                 raise
+
+
+def batch_iter(iterable, size):
+    """batch an iterable up into tuples with a maximum size
+
+    Args:
+        iterable (iterable): the iterable to slice
+        size (int): the maximum batch size
+
+    Returns:
+        an iterator over the chunks
+    """
+    # make sure we can deal with iterables like lists too
+    sourceiter = iter(iterable)
+    # call islice until it returns an empty tuple
+    return iter(lambda: tuple(islice(sourceiter, size)), ())