Merge branch 'master' of github.com:matrix-org/synapse into sql_refactor

Conflicts:
	synapse/storage/roommember.py

10 files changed, 330 insertions, 88 deletions

diff --git a/nuke-room-from-db.sh b/nuke-room-from-db.sh
new file mode 100755
index 0000000000..58c036c896
--- /dev/null
+++ b/nuke-room-from-db.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+## CAUTION:
+## This script will remove (hopefully) all trace of the given room ID from
+## your homeserver.db
+
+## Do not run it lightly.
+
+ROOMID="$1"
+
+sqlite3 homeserver.db <<EOF
+DELETE FROM context_depth WHERE context = '$ROOMID';
+DELETE FROM current_state WHERE context = '$ROOMID';
+DELETE FROM feedback WHERE room_id = '$ROOMID';
+DELETE FROM messages WHERE room_id = '$ROOMID';
+DELETE FROM pdu_backward_extremities WHERE context = '$ROOMID';
+DELETE FROM pdu_edges WHERE context = '$ROOMID';
+DELETE FROM pdu_forward_extremities WHERE context = '$ROOMID';
+DELETE FROM pdus WHERE context = '$ROOMID';
+DELETE FROM room_data WHERE room_id = '$ROOMID';
+DELETE FROM room_memberships WHERE room_id = '$ROOMID';
+DELETE FROM rooms WHERE room_id = '$ROOMID';
+DELETE FROM state_pdus WHERE context = '$ROOMID';
+EOF
diff --git a/webclient/app-controller.js b/webclient/app-controller.js
index 086fa3d946..7fa87e30c1 100644
--- a/webclient/app-controller.js
+++ b/webclient/app-controller.js
@@ -21,8 +21,8 @@ limitations under the License.
 'use strict';
 
 angular.module('MatrixWebClientController', ['matrixService'])
-.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', 'matrixService',
-                               function($scope, $location, $rootScope, matrixService) {
+.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', 'matrixService', 'eventStreamService',
+                               function($scope, $location, $rootScope, matrixService, eventStreamService) {
          
     // Check current URL to avoid to display the logout button on the login page
     $scope.location = $location.path();
@@ -44,11 +44,17 @@ angular.module('MatrixWebClientController', ['matrixService'])
         else {
             $scope.config = matrixService.config();        
         }
-    };    
-    
+    };
+
+    if (matrixService.config()) {
+        eventStreamService.resume();
+    }
     
     // Logs the user out 
     $scope.logout = function() {
+        // kill the event stream
+        eventStreamService.stop();
+    
         // Clean permanent data
         matrixService.setConfig({});
         matrixService.saveConfig();
@@ -57,7 +63,7 @@ angular.module('MatrixWebClientController', ['matrixService'])
         $location.path("login");
     };
 
-    // Listen to the event indicating that the access token is no more valid.
+    // Listen to the event indicating that the access token is no longer valid.
     // In this case, the user needs to log in again.
     $scope.$on("M_UNKNOWN_TOKEN", function() {
         console.log("Invalid access token -> log user out");
@@ -65,4 +71,4 @@ angular.module('MatrixWebClientController', ['matrixService'])
     });
 }]);
 
-   
\ No newline at end of file
+   
diff --git a/webclient/app.js b/webclient/app.js
index 0b613fa206..6e0351067f 100644
--- a/webclient/app.js
+++ b/webclient/app.js
@@ -20,7 +20,9 @@ var matrixWebClient = angular.module('matrixWebClient', [
     'LoginController',
     'RoomController',
     'RoomsController',
-    'matrixService'
+    'matrixService',
+    'eventStreamService',
+    'eventHandlerService'
 ]);
 
 matrixWebClient.config(['$routeProvider', '$provide', '$httpProvider',
@@ -59,12 +61,16 @@ matrixWebClient.config(['$routeProvider', '$provide', '$httpProvider',
         $httpProvider.interceptors.push('AccessTokenInterceptor');
     }]);
 
-matrixWebClient.run(['$location', 'matrixService' , function($location, matrixService) {
+matrixWebClient.run(['$location', 'matrixService', 'eventStreamService', function($location, matrixService, eventStreamService) {
     // If we have no persistent login information, go to the login page
     var config = matrixService.config();
     if (!config || !config.access_token) {
+        eventStreamService.stop();
         $location.path("login");
     }
+    else {
+        eventStreamService.resume();
+    }
 }]);
 
 matrixWebClient
diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js
new file mode 100644
index 0000000000..f7411fd80a
--- /dev/null
+++ b/webclient/components/matrix/event-handler-service.js
@@ -0,0 +1,109 @@
+/*
+Copyright 2014 matrix.org
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+'use strict';
+
+/*
+This service handles what should happen when you get an event. This service does
+not care where the event came from, it only needs enough context to be able to 
+process them. Events may be coming from the event stream, the REST API (via 
+direct GETs or via a pagination stream API), etc.
+
+Typically, this service will store events or broadcast them to any listeners
+(e.g. controllers) via $broadcast. Alternatively, it may update the $rootScope
+if typically all the $on method would do is update its own $scope.
+*/
+angular.module('eventHandlerService', [])
+.factory('eventHandlerService', ['matrixService', '$rootScope', function(matrixService, $rootScope) {
+    var MSG_EVENT = "MSG_EVENT";
+    var MEMBER_EVENT = "MEMBER_EVENT";
+    var PRESENCE_EVENT = "PRESENCE_EVENT";
+    
+    $rootScope.events = {
+        rooms: {}, // will contain roomId: { messages:[], members:[] }
+    };
+    
+    var initRoom = function(room_id) {
+        console.log("Creating new handler entry for " + room_id);
+        $rootScope.events.rooms[room_id] = {};
+        $rootScope.events.rooms[room_id].messages = [];
+        $rootScope.events.rooms[room_id].members = [];
+    }
+    
+    var handleMessage = function(event, isLiveEvent) {
+        if ("membership_target" in event.content) {
+            event.user_id = event.content.membership_target;
+        }
+        if (!(event.room_id in $rootScope.events.rooms)) {
+            initRoom(event.room_id);
+        }
+        
+        if (isLiveEvent) {
+            $rootScope.events.rooms[event.room_id].messages.push(event);
+        }
+        else {
+            $rootScope.events.rooms[event.room_id].messages.unshift(event);
+        }
+        
+        // TODO send delivery receipt if isLiveEvent
+        
+        // $broadcast this, as controllers may want to do funky things such as
+        // scroll to the bottom, etc which cannot be expressed via simple $scope
+        // updates.
+        $rootScope.$broadcast(MSG_EVENT, event, isLiveEvent);
+    };
+    
+    var handleRoomMember = function(event, isLiveEvent) {
+        $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent);
+    };
+    
+    var handlePresence = function(event, isLiveEvent) {
+        $rootScope.$broadcast(PRESENCE_EVENT, event, isLiveEvent);
+    };
+    
+    
+    return {
+        MSG_EVENT: MSG_EVENT,
+        MEMBER_EVENT: MEMBER_EVENT,
+        PRESENCE_EVENT: PRESENCE_EVENT,
+        
+    
+        handleEvent: function(event, isLiveEvent) {
+            switch(event.type) {
+                case "m.room.message":
+                    handleMessage(event, isLiveEvent);
+                    break;
+                case "m.room.member":
+                    handleRoomMember(event, isLiveEvent);
+                    break;
+                case "m.presence":
+                    handlePresence(event, isLiveEvent);
+                    break;
+                default:
+                    console.log("Unable to handle event type " + event.type);
+                    break;
+            }
+        },
+        
+        // isLiveEvents determines whether notifications should be shown, whether
+        // messages get appended to the start/end of lists, etc.
+        handleEvents: function(events, isLiveEvents) {
+            for (var i=0; i<events.length; i++) {
+                this.handleEvent(events[i], isLiveEvents);
+            }
+        }
+    };
+}]);
diff --git a/webclient/components/matrix/event-stream-service.js b/webclient/components/matrix/event-stream-service.js
new file mode 100644
index 0000000000..9f678e8454
--- /dev/null
+++ b/webclient/components/matrix/event-stream-service.js
@@ -0,0 +1,131 @@
+/*
+Copyright 2014 matrix.org
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+'use strict';
+
+/*
+This service manages where in the event stream the web client currently is,
+repolling the event stream, and provides methods to resume/pause/stop the event 
+stream. This service is not responsible for parsing event data. For that, see 
+the eventHandlerService.
+*/
+angular.module('eventStreamService', [])
+.factory('eventStreamService', ['$q', '$timeout', 'matrixService', 'eventHandlerService', function($q, $timeout, matrixService, eventHandlerService) {
+    var END = "END";
+    var START = "START";
+    var TIMEOUT_MS = 5000;
+    var ERR_TIMEOUT_MS = 5000;
+    
+    var settings = {
+        from: "END",
+        to: undefined,
+        limit: undefined,
+        shouldPoll: true,
+        isActive: false
+    };
+    
+    // interrupts the stream. Only valid if there is a stream conneciton 
+    // open.
+    var interrupt = function(shouldPoll) {
+        console.log("[EventStream] interrupt("+shouldPoll+") "+
+                    JSON.stringify(settings));
+        settings.shouldPoll = shouldPoll;
+        settings.isActive = false;
+    };
+    
+    var saveStreamSettings = function() {
+        localStorage.setItem("streamSettings", JSON.stringify(settings));
+    };
+    
+    var startEventStream = function() {
+        settings.shouldPoll = true;
+        settings.isActive = true;
+        var deferred = $q.defer();
+        // run the stream from the latest token
+        matrixService.getEventStream(settings.from, TIMEOUT_MS).then(
+            function(response) {
+                if (!settings.isActive) {
+                    console.log("[EventStream] Got response but now inactive. Dropping data.");
+                    return;
+                }
+                
+                settings.from = response.data.end;
+                
+                console.log("[EventStream] Got response from "+settings.from+" to "+response.data.end);
+                eventHandlerService.handleEvents(response.data.chunk, true);
+                
+                deferred.resolve(response);
+                
+                if (settings.shouldPoll) {
+                    $timeout(startEventStream, 0);
+                }
+                else {
+                    console.log("[EventStream] Stopping poll.");
+                }
+            },
+            function(error) {
+                if (error.status == 403) {
+                    settings.shouldPoll = false;
+                }
+                
+                deferred.reject(error);
+                
+                if (settings.shouldPoll) {
+                    $timeout(startEventStream, ERR_TIMEOUT_MS);
+                }
+                else {
+                    console.log("[EventStream] Stopping polling.");
+                }
+            }
+        );
+        return deferred.promise;
+    };
+    
+    return {
+        // resume the stream from whereever it last got up to. Typically used
+        // when the page is opened.
+        resume: function() {
+            if (settings.isActive) {
+                console.log("[EventStream] Already active, ignoring resume()");
+                return;
+            }
+        
+            console.log("[EventStream] resume "+JSON.stringify(settings));
+            return startEventStream();
+        },
+        
+        // pause the stream. Resuming it will continue from the current position
+        pause: function() {
+            console.log("[EventStream] pause "+JSON.stringify(settings));
+            // kill any running stream
+            interrupt(false);
+            // save the latest token
+            saveStreamSettings();
+        },
+        
+        // stop the stream and wipe the position in the stream. Typically used
+        // when logging out / logged out.
+        stop: function() {
+            console.log("[EventStream] stop "+JSON.stringify(settings));
+            // kill any running stream
+            interrupt(false);
+            // clear the latest token
+            settings.from = END;
+            saveStreamSettings();
+        }
+    };
+
+}]);
diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js
index 6d66111469..0cc85db28e 100644
--- a/webclient/components/matrix/matrix-service.js
+++ b/webclient/components/matrix/matrix-service.js
@@ -16,6 +16,12 @@ limitations under the License.
 
 'use strict';
 
+/*
+This service wraps up Matrix API calls. 
+
+This serves to isolate the caller from changes to the underlying url paths, as
+well as attach common params (e.g. access_token) to requests.
+*/
 angular.module('matrixService', [])
 .factory('matrixService', ['$http', '$q', '$rootScope', function($http, $q, $rootScope) {
         
@@ -36,10 +42,16 @@ angular.module('matrixService', [])
     var MAPPING_PREFIX = "alias_for_";
 
     var doRequest = function(method, path, params, data) {
+        if (!config) {
+            console.warn("No config exists. Cannot perform request to "+path);
+            return;
+        }
+    
         // Inject the access token
         if (!params) {
             params = {};
         }
+        
         params.access_token = config.access_token;
         
         return doBaseRequest(config.homeserver, method, path, params, data, undefined);
@@ -297,6 +309,15 @@ angular.module('matrixService', [])
             return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); 
         },
         
+        // start listening on /events
+        getEventStream: function(from, timeout) {
+            var path = "/events";
+            var params = {
+                from: from,
+                timeout: timeout
+            };
+            return doRequest("GET", path, params);
+        },
         
         // 
         testLogin: function() {
diff --git a/webclient/index.html b/webclient/index.html
index e62ec39669..31b62efaa8 100644
--- a/webclient/index.html
+++ b/webclient/index.html
@@ -14,6 +14,8 @@
     <script src="room/room-controller.js"></script>
     <script src="rooms/rooms-controller.js"></script>
     <script src="components/matrix/matrix-service.js"></script>
+    <script src="components/matrix/event-stream-service.js"></script>
+    <script src="components/matrix/event-handler-service.js"></script>
     <script src="components/fileInput/file-input-directive.js"></script>
     <script src="components/fileUpload/file-upload-service.js"></script>
 </head>
diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js
index 8bd6a4e84f..aa928ef48d 100644
--- a/webclient/login/login-controller.js
+++ b/webclient/login/login-controller.js
@@ -1,6 +1,6 @@
 angular.module('LoginController', ['matrixService'])
-.controller('LoginController', ['$scope', '$location', 'matrixService',
-                                    function($scope, $location, matrixService) {
+.controller('LoginController', ['$scope', '$location', 'matrixService', 'eventStreamService',
+                                    function($scope, $location, matrixService, eventStreamService) {
     'use strict';
     
     
@@ -51,7 +51,7 @@ angular.module('LoginController', ['matrixService'])
 
                 // And permanently save it
                 matrixService.saveConfig();
-
+                eventStreamService.resume();
                  // Go to the user's rooms list page
                 $location.path("rooms");
             },
@@ -83,6 +83,7 @@ angular.module('LoginController', ['matrixService'])
                         access_token: response.data.access_token
                     });
                     matrixService.saveConfig();
+                    eventStreamService.resume();
                     $location.path("rooms");
                 }
                 else {
diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js
index fb6e2025fc..0d54c6f4d8 100644
--- a/webclient/room/room-controller.js
+++ b/webclient/room/room-controller.js
@@ -15,8 +15,8 @@ limitations under the License.
 */
 
 angular.module('RoomController', [])
-.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService',
-                               function($scope, $http, $timeout, $routeParams, $location, matrixService) {
+.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService', 'eventStreamService', 'eventHandlerService',
+                               function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService) {
    'use strict';
     var MESSAGES_PER_PAGINATION = 10;
     $scope.room_id = $routeParams.room_id;
@@ -28,9 +28,7 @@ angular.module('RoomController', [])
         can_paginate: true, // this is toggled off when we run out of items
         stream_failure: undefined // the response when the stream fails
     };
-    $scope.messages = [];
     $scope.members = {};
-    $scope.stopPoll = false;
 
     $scope.imageURLToSend = "";
     $scope.userIDToInvite = "";
@@ -42,34 +40,24 @@ angular.module('RoomController', [])
         },0);
     };
     
-    var parseChunk = function(chunks, appendToStart) {
-        for (var i = 0; i < chunks.length; i++) {
-            var chunk = chunks[i];
-            if (chunk.room_id == $scope.room_id && chunk.type == "m.room.message") {
-                if ("membership_target" in chunk.content) {
-                    chunk.user_id = chunk.content.membership_target;
-                }
-                if (appendToStart) {
-                    $scope.messages.unshift(chunk);
-                }
-                else {
-                    $scope.messages.push(chunk);
-                    scrollToBottom();
-                }
-            }
-            else if (chunk.room_id == $scope.room_id && chunk.type == "m.room.member") {
-                updateMemberList(chunk);
-            }
-            else if (chunk.type === "m.presence") {
-                updatePresence(chunk);
-            }
+    $scope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) {
+        if (isLive && event.room_id === $scope.room_id) {
+            scrollToBottom();
         }
-    };
+    });
+    
+    $scope.$on(eventHandlerService.MEMBER_EVENT, function(ngEvent, event, isLive) {
+        updateMemberList(event);
+    });
+    
+    $scope.$on(eventHandlerService.PRESENCE_EVENT, function(ngEvent, event, isLive) {
+        updatePresence(event);
+    });
     
     var paginate = function(numItems) {
         matrixService.paginateBackMessages($scope.room_id, $scope.state.earliest_token, numItems).then(
             function(response) {
-                parseChunk(response.data.chunk, true);
+                eventHandlerService.handleEvents(response.data.chunk, false);
                 $scope.state.earliest_token = response.data.end;
                 if (response.data.chunk.length < MESSAGES_PER_PAGINATION) {
                     // no more messages to paginate :(
@@ -82,43 +70,6 @@ angular.module('RoomController', [])
         )
     };
 
-    var shortPoll = function() {
-        $http.get(matrixService.config().homeserver + matrixService.prefix + "/events", {
-            "params": {
-                "access_token": matrixService.config().access_token,
-                "from": $scope.state.events_from,
-                "timeout": 5000
-            }})
-            .then(function(response) {
-                $scope.state.stream_failure = undefined;
-                console.log("Got response from "+$scope.state.events_from+" to "+response.data.end);
-                $scope.state.events_from = response.data.end;
-                $scope.feedback = "";
-
-                parseChunk(response.data.chunk, false);
-                
-                if ($scope.stopPoll) {
-                    console.log("Stopping polling.");
-                }
-                else {
-                    $timeout(shortPoll, 0);
-                }
-            }, function(response) {
-                $scope.state.stream_failure = response;
-
-                if (response.status == 403) {
-                    $scope.stopPoll = true;
-                }
-                
-                if ($scope.stopPoll) {
-                    console.log("Stopping polling.");
-                }
-                else {
-                    $timeout(shortPoll, 5000);
-                }
-            });
-    };
-
     var updateMemberList = function(chunk) {
         var isNewMember = !(chunk.target_user_id in $scope.members);
         if (isNewMember) {
@@ -133,7 +84,6 @@ angular.module('RoomController', [])
                     function(response) {
                         var member = $scope.members[chunk.target_user_id];
                         if (member !== undefined) {
-                            console.log("Updated displayname "+chunk.target_user_id+" to " + response.data.displayname);
                             member.displayname = response.data.displayname;
                         }
                     }
@@ -142,7 +92,6 @@ angular.module('RoomController', [])
                     function(response) {
                          var member = $scope.members[chunk.target_user_id];
                          if (member !== undefined) {
-                            console.log("Updated image for "+chunk.target_user_id+" to " + response.data.avatar_url);
                             member.avatar_url = response.data.avatar_url;
                          }
                     }
@@ -218,8 +167,6 @@ angular.module('RoomController', [])
         matrixService.join($scope.room_id).then(
             function() {
                 console.log("Joined room "+$scope.room_id);
-                // Now start reading from the stream
-                $timeout(shortPoll, 0);
 
                 // Get the current member list
                 matrixService.getMemberList($scope.room_id).then(
@@ -278,9 +225,4 @@ angular.module('RoomController', [])
     $scope.loadMoreHistory = function() {
         paginate(MESSAGES_PER_PAGINATION);
     };
-
-    $scope.$on('$destroy', function(e) {
-        console.log("onDestroyed: Stopping poll.");
-        $scope.stopPoll = true;
-    });
 }]);
diff --git a/webclient/room/room.html b/webclient/room/room.html
index 3b9ba713de..93917071d9 100644
--- a/webclient/room/room.html
+++ b/webclient/room/room.html
@@ -22,14 +22,14 @@
     
     <div class="messageTableWrapper">
         <table class="messageTable">
-            <tr ng-repeat="msg in messages" ng-class="msg.user_id === state.user_id ? 'mine' : ''">
+            <tr ng-repeat="msg in events.rooms[room_id].messages" ng-class="msg.user_id === state.user_id ? 'mine' : ''">
                 <td class="leftBlock">
-                    <div class="sender" ng-hide="messages[$index - 1].user_id === msg.user_id">{{ members[msg.user_id].displayname || msg.user_id }}</div>
+                    <div class="sender" ng-hide="events.rooms[room_id].messages[$index - 1].user_id === msg.user_id">{{ members[msg.user_id].displayname || msg.user_id }}</div>
                     <div class="timestamp">{{ msg.content.hsob_ts | date:'MMM d HH:mm:ss' }}</div>
                 </td>
                 <td class="avatar">
                     <img ng-src="{{ members[msg.user_id].avatar_url || 'img/default-profile.jpg' }}" width="32" height="32"
-                         ng-hide="messages[$index - 1].user_id === msg.user_id || msg.user_id === state.user_id"/>
+                         ng-hide="events.rooms[room_id].messages[$index - 1].user_id === msg.user_id || msg.user_id === state.user_id"/>
                 </td>
                 <td ng-class="!msg.content.membership_target ? (msg.content.msgtype === 'm.emote' ? 'emote text' : 'text') : ''">
                     <div class="bubble">
@@ -40,7 +40,7 @@
                 </td>
                 <td class="rightBlock">
                     <img ng-src="{{ members[msg.user_id].avatar_url || 'img/default-profile.jpg' }}" width="32" height="32"
-                         ng-hide="messages[$index - 1].user_id === msg.user_id || msg.user_id !== state.user_id"/>
+                         ng-hide="events.rooms[room_id].messages[$index - 1].user_id === msg.user_id || msg.user_id !== state.user_id"/>
                 </td>
             </tr>
         </table>