optimize index for rating history

update log server to prevent delays or missed information

GameLogServer/GameLogServer/log_reader.py

@@ -1,6 +1,8 @@
 import re
 import os
 import time
+import random
+import string
 
 class LogReader(object):
     def __init__(self):
@@ -8,7 +10,7 @@ class LogReader(object):
         # (if the time between checks is greater, ignore ) - in seconds
         self.max_file_time_change = 60
 
-    def read_file(self, path):
+    def read_file(self, path, retrieval_key):
         # this removes old entries that are no longer valid
         try:
             self._clear_old_logs()
@@ -22,57 +24,95 @@ class LogReader(object):
 
         # prevent traversing directories
         if re.search('r^.+\.\.\\.+$', path):
-            return False
+            return self._generate_bad_response()
+
         # must be a valid log path and log file
         if not re.search(r'^.+[\\|\/](.+)[\\|\/].+.log$', path):
-            return False
+            return self._generate_bad_response()
 
         # get the new file size
         new_file_size = self.file_length(path)
           
         # the log size was unable to be read (probably the wrong path)
         if new_file_size < 0:
-            return False
+            return self._generate_bad_response()
 
-        # this is the first time the log has been requested
-        if path not in self.log_file_sizes:
-            self.log_file_sizes[path] = { 
-                'length' : new_file_size,
-                'read': time.time()
+        next_retrieval_key = self._generate_key()
+
+        #print('next key is %s' % next_retrieval_key)
+
+        # this is the first time the key has been requested, so we need to the next one
+        if retrieval_key not in self.log_file_sizes:
+            print ('requested key %s does not exist' % retrieval_key)
+            self.log_file_sizes[next_retrieval_key] = { 
+                'size': new_file_size,
+                'length': 0,
+                'read': time.time(),
+                'next_key': next_retrieval_key, # the key that IW4MAdmin should request next
+                'previous_key': retrieval_key # the old key that can be deleted next
             }
-            return ''
+            return {
+                'content': None,
+                'next_key': next_retrieval_key
+            }
+        else:
+            current_log_info = self.log_file_sizes[retrieval_key]
 
         # grab the previous values
-        last_length = self.log_file_sizes[path]['length']
-        file_size_difference = new_file_size - last_length
+        last_length = current_log_info['length']
+        last_size = current_log_info['size']
+        file_size_difference = new_file_size - last_size
+
+        #print('generating info for next key %s' % next_retrieval_key)
 
         # update the new size and actually read the data
-        self.log_file_sizes[path] = {
-            'length': new_file_size,
-            'read': time.time()
+        self.log_file_sizes[next_retrieval_key] = {
+            'size' : new_file_size,
+            'length': last_length,
+            'read': time.time(),
+            'next_key': next_retrieval_key,
+            'previous_key': retrieval_key
         }
 
-        new_log_info = self.get_file_lines(path, file_size_difference)
-        return new_log_info
+        if current_log_info['previous_key'] in self.log_file_sizes.keys():
+            #print('deleting old key %s' % current_log_info['previous_key'])
+            del self.log_file_sizes[current_log_info['previous_key']]
 
-    def get_file_lines(self, path, length):
+        #print('reading %i bytes starting at %i' % (file_size_difference, last_size))
+
+        new_log_info = self.get_file_lines(path, last_size, file_size_difference)
+        return {
+            'content': new_log_info,
+            'next_key': next_retrieval_key
+        }
+
+    def get_file_lines(self, path, start_position, length_to_read):
         try:
             file_handle = open(path, 'rb')
-            file_handle.seek(-length, 2)
-            file_data = file_handle.read(length)
+            file_handle.seek(start_position)
+            file_data = file_handle.read(length_to_read)
             file_handle.close()
             # using ignore errors omits the pesky 0xb2 bytes we're reading in for some reason
             return file_data.decode('utf-8', errors='ignore')
         except Exception as e:
-            print('could not read the log file at {0}, wanted to read {1} bytes'.format(path, length))
+            print('could not read the log file at {0}, wanted to read {1} bytes'.format(path, length_to_read))
             print(e)
             return False
 
     def _clear_old_logs(self):
         expired_logs = [path for path in self.log_file_sizes if int(time.time() - self.log_file_sizes[path]['read']) > self.max_file_time_change]
-        for log in expired_logs:
-            print('removing expired log {0}'.format(log))
-            del self.log_file_sizes[log]
+        for key in expired_logs:
+            print('removing expired log with key {0}'.format(key))
+            del self.log_file_sizes[key]
+
+    def _generate_bad_response(self):
+        return {
+            'content': None,
+            'next_key': None
+        }
+
+    def _generate_key(self):
+        return ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
         
     def file_length(self, path):
         try:

GameLogServer/GameLogServer/log_resource.py

@@ -3,12 +3,14 @@ from GameLogServer.log_reader import reader
 from base64 import urlsafe_b64decode
 
 class LogResource(Resource):
-    def get(self, path):
+    def get(self, path, retrieval_key):
         path = urlsafe_b64decode(path).decode('utf-8')
-        log_info = reader.read_file(path)
+        log_info = reader.read_file(path, retrieval_key)
+        content = log_info['content']
 
         return {
-            'success' : log_info is not False,
-            'length':  0 if log_info is False else len(log_info),
-            'data': log_info
+            'success' : content is not None,
+            'length':  0 if content is None else len(content),
+            'data': content,
+            'next_key': log_info['next_key']
         }

GameLogServer/GameLogServer/server.py

@@ -10,5 +10,5 @@ def init():
     log = logging.getLogger('werkzeug')
     log.setLevel(logging.ERROR)
     api = Api(app)
-    api.add_resource(LogResource, '/log/<string:path>')
+    api.add_resource(LogResource, '/log/<string:path>/<string:retrieval_key>')
     #api.add_resource(RestartResource, '/restart')