Compress alerts with zlib level 1 to fit in sheriff-o-matic's memcache.

When there were 2000 failures recently the resulting alerts JSON was
~2.5MB. This is too big to fit in memcache's 1MB limit [1].

However the data compresses readily (sizes in K; level 0 is the
uncompressed size):

>>> import zlib
>>> uncompressed = open('data.json', 'r').read()
>>> for level in range(0,10):
...   compressed = zlib.compress(uncompressed, level)
...   print level, len(compressed) / 1024.0
...
0 2525.62207031
1 129.721679688
2 121.662109375
3 120.194335938
4 117.762695312
5 111.912109375
6 105.26953125
7 104.64453125
8 101.791015625
9 101.745117188

[1] https://developers.google.com/appengine/docs/python/memcache/#Python_Limits

TEST=PYTHONPATH=/usr/local/google_appengine Tools/GardeningServer/tests.py
BUG=401358
R=eseidel@chromium.org, ojan@chromium.org

Review URL: https://codereview.chromium.org/469983002

git-svn-id: svn://svn.chromium.org/blink/trunk@180222 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/Tools/GardeningServer/alerts.py

@@ -6,6 +6,7 @@ import calendar
 import datetime
 import json
 import webapp2
+import zlib
 
 from google.appengine.api import memcache
 
@@ -24,10 +25,11 @@ class AlertsHandler(webapp2.RequestHandler):
     def get(self):
         self.response.headers.add_header('Access-Control-Allow-Origin', '*')
         self.response.headers['Content-Type'] = 'application/json'
-        alerts = memcache.get(AlertsHandler.MEMCACHE_ALERTS_KEY)
-        if not alerts:
+        compressed = memcache.get(AlertsHandler.MEMCACHE_ALERTS_KEY)
+        if not compressed:
             return
-        self.response.write(json.dumps(alerts, cls=DateTimeEncoder, indent=1))
+        uncompressed = zlib.decompress(compressed)
+        self.response.write(uncompressed)
 
     def post(self):
         try:
@@ -39,7 +41,10 @@ class AlertsHandler(webapp2.RequestHandler):
             'date': datetime.datetime.utcnow(),
             'alerts': alerts['alerts']
         })
-        memcache.set(AlertsHandler.MEMCACHE_ALERTS_KEY, alerts)
+        uncompressed = json.dumps(alerts, cls=DateTimeEncoder, indent=1)
+        compression_level = 1
+        compressed = zlib.compress(uncompressed, compression_level)
+        memcache.set(AlertsHandler.MEMCACHE_ALERTS_KEY, compressed)
 
 
 app = webapp2.WSGIApplication([

third_party/WebKit/Tools/GardeningServer/alerts_test.py

@@ -4,6 +4,8 @@
 
 import alerts
 import json
+import random
+import string
 import unittest
 import webtest
 
@@ -62,3 +64,69 @@ class AlertsTest(unittest.TestCase):
         self.check_json_headers(res)
         alerts = json.loads(res.body)
         self.assertEqual(alerts['alerts'], 'everything is OK')
+
+    def test_large_number_of_alerts(self):
+        # This generates ~2.5MB of JSON that compresses to ~750K. Real
+        # data compresses about 6x better.
+        random.seed(0xf00f00)
+        put_alerts = self.generate_fake_alerts(4000)
+
+        params = {'content': json.dumps(put_alerts)}
+        self.testapp.post('/alerts', params)
+
+        res = self.testapp.get('/alerts')
+        got_alerts = json.loads(res.body)
+        self.assertEquals(got_alerts['alerts'], put_alerts['alerts'])
+
+    def generate_fake_alerts(self, n):
+        return {'alerts': [self.generate_fake_alert() for _ in range(n)]}
+
+    def generate_fake_alert(self):
+        # fake labels
+        labels = [['', 'last_', 'latest_', 'failing_', 'passing_'],
+                  ['build', 'builder', 'revision'],
+                  ['', 's', '_url', '_reason', '_name']]
+
+        def label():
+            return string.join(map(random.choice, labels), '')
+
+        # fake values
+        def time():
+            return random.randint(1407976107614, 1408076107614) / 101.0
+
+        def build():
+            return random.randint(2737, 2894)
+
+        def revision():
+            return random.randint(288849, 289415)
+
+        tests = [['Activity', 'Async', 'Browser', 'Content', 'Input'],
+                 ['Manager', 'Card', 'Sandbox', 'Container'],
+                 ['Test.'],
+                 ['', 'Basic', 'Empty', 'More'],
+                 ['Mouse', 'App', 'Selection', 'Network', 'Grab'],
+                 ['Input', 'Click', 'Failure', 'Capture']]
+
+        def test():
+            return string.join(map(random.choice, tests), '')
+
+        def literal_array():
+            generator = random.choice([time, build, revision])
+            return [generator() for _ in range(random.randint(0, 10))]
+
+        def literal_map():
+            generators = [build, revision, test, literal_array]
+            obj = {}
+            for _ in range(random.randint(3, 9)):
+                obj[label()] = random.choice(generators)()
+            return obj
+
+        def value():
+            generators = [time, build, revision, test, literal_array,
+                          literal_map]
+            return random.choice(generators)()
+
+        alert = {}
+        for _ in range(random.randint(6, 9)):
+            alert[label()] = value()
+        return alert