Add beanstalk integration along with tests

Beanstalk integration uses webhooks that use http basic auth to authenticate
the sending user.

(imported from commit bd65f5b2d052a3c1eb04da64d055a3640a384892)

humbug/urls.py

@@ -143,6 +143,7 @@ urlpatterns += patterns('zephyr.views',
     # These are integration-specific web hook callbacks
     url(r'^api/v1/external/github$',        'api_github_landing'),
     url(r'^api/v1/external/jira/(\w+)/?$',  'api_jira_webhook'),
+    url(r'^api/v1/external/beanstalk$',     'api_beanstalk_webhook'),
 )
 
 urlpatterns += patterns('zephyr.tornadoviews',

zephyr/fixtures/beanstalk/beanstalk_git_multiple.json

@@ -0,0 +1 @@
+{ "after": "20098158e20ae4257ca228e18b1de8205a69604d", "before": "e50508df24cee0c6e6b1c051ce348282ea152cb3", "branch": "master", "commits": [ { "author": { "email": "lfranchi@kde.org", "name": "Leo Franchi" }, "changed_dirs": [], "changed_files": [ [ "new-file", "add" ] ], "id": "edf529c7a64d44937e534fe6e78323e79a160b13", "message": "Added new file", "timestamp": "2013-04-01T19:38:40Z", "url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/edf529c7" }, { "author": { "email": "lfranchi@kde.org", "name": "Leo Franchi" }, "changed_dirs": [], "changed_files": [ [ "new-file", "edit" ] ], "id": "c2a191b9e79208c4a9aa5efda917a210f34f61d6", "message": "Filled in new file with some stuff", "timestamp": "2013-04-01T19:38:54Z", "url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/c2a191b9" }, { "author": { "email": "lfranchi@kde.org", "name": "Leo Franchi" }, "changed_dirs": [], "changed_files": [ [ "new-file", "edit" ], [ "work-test.py", "edit" ] ], "id": "20098158e20ae4257ca228e18b1de8205a69604d", "message": "More work to fix some bugs\n\nSecond line of commit message, yes plzzzz", "timestamp": "2013-04-01T19:39:08Z", "url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/20098158" } ], "push_is_too_large": false, "pusher_id": 348341, "pusher_name": "Leo Franchi", "ref": "refs/heads/master", "repository": { "name": "work-test", "owner": { "email": "lfranchi@gmail.com", "name": "Leo Franchi" }, "private": true, "url": "http://lfranchi-svn.beanstalkapp.com/work-test" }, "uri": "git@lfranchi-svn.beanstalkapp.com:/work-test.git" }

zephyr/fixtures/beanstalk/beanstalk_git_singlecommit.json

@@ -0,0 +1 @@
+{ "after": "e50508df24cee0c6e6b1c051ce348282ea152cb3", "before": "d190b0f5f1873f809db4050f0b31e89d349381f1", "branch": "master", "commits": [ { "author": { "email": "lfranchi@kde.org", "name": "Leo Franchi" }, "changed_dirs": [], "changed_files": [ [ "work-test.py", "edit" ] ], "id": "e50508df24cee0c6e6b1c051ce348282ea152cb3", "message": "add some stuff", "timestamp": "2013-04-01T19:21:11Z", "url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/e50508df" } ], "push_is_too_large": false, "pusher_id": 348341, "pusher_name": "Leo Franchi", "ref": "refs/heads/master", "repository": { "name": "work-test", "owner": { "email": "lfranchi@gmail.com", "name": "Leo Franchi" }, "private": true, "url": "http://lfranchi-svn.beanstalkapp.com/work-test" }, "uri": "git@lfranchi-svn.beanstalkapp.com:/work-test.git" }

zephyr/fixtures/beanstalk/beanstalk_svn_addremove.json

@@ -0,0 +1 @@
+{ "author": "lfranchi", "author_email": "lfranchi@gmail.com", "author_full_name": "Leo Franchi", "changed_dirs": [], "changed_files": [ [ "new-file.txt", "add" ], [ "lots-of-work.py", "delete" ] ], "changeset_url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/3", "message": "Removed a file and added another one!\n\nTHis is a pretty long commit", "revision": 3, "time": "2013/04/01 19:03:50 +0000" }

zephyr/fixtures/beanstalk/beanstalk_svn_changefile.json

@@ -0,0 +1 @@
+{ "author": "lfranchi", "author_email": "lfranchi@gmail.com", "author_full_name": "Leo Franchi", "changed_dirs": [], "changed_files": [ [ "lots-of-work.py", "edit" ] ], "changeset_url": "http://lfranchi-svn.beanstalkapp.com/work-test/changesets/2", "message": "Added some code", "revision": 2, "time": "2013/04/01 19:01:33 +0000" }

zephyr/tests.py

@@ -158,6 +158,9 @@ class AuthedTestCase(TestCase):
     def assert_json_error_contains(self, result, msg_substring):
         self.assertIn(msg_substring, self.get_json_error(result))
 
+    def fixture_jsondata(self, type, action):
+        return open(os.path.join(os.path.dirname(__file__),
+                                 "fixtures/%s/%s_%s.json" % (type, type, action,))).read()
 class PublicURLTest(TestCase):
     """
     Account creation URLs are accessible even when not logged in. Authenticated
@@ -2397,10 +2400,6 @@ class StarTests(AuthedTestCase):
 class JiraHookTests(AuthedTestCase):
     fixtures = ['messages.json']
 
-    def fixture_data(self, action):
-        return open(os.path.join(os.path.dirname(__file__),
-                                 "fixtures/jira/jira_%s.json" % (action,))).read()
-
     def send_jira_message(self, action):
         email = "hamlet@humbughq.com"
         api_key = self.get_api_key(email)
@@ -2409,7 +2408,7 @@ class JiraHookTests(AuthedTestCase):
         user_profile = self.get_user_profile(email)
         do_add_subscription(user_profile, stream, no_log=True)
 
-        result = self.client.post("/api/v1/external/jira/%s/" % api_key, self.fixture_data(action),
+        result = self.client.post("/api/v1/external/jira/%s/" % api_key, self.fixture_jsondata('jira', action),
                                   content_type="application/json")
         self.assert_json_success(result)
 
@@ -2468,6 +2467,66 @@ class JiraHookTests(AuthedTestCase):
 
 > Fixed it, finally!""")
 
+class BeanstalkHookTests(AuthedTestCase):
+    fixtures = ['messages.json']
+
+    def http_auth(self, username, password):
+        import base64
+        credentials = base64.b64encode('%s:%s' % (username, password))
+        auth_string = 'Basic %s' % credentials
+        return auth_string
+
+    def send_beanstalk_message(self, action):
+        email = "hamlet@humbughq.com"
+        api_key = self.get_api_key(email)
+        stream, _ = create_stream_if_needed(Realm.objects.get(domain="humbughq.com"), 'commits')
+        user_profile = self.get_user_profile(email)
+        do_add_subscription(user_profile, stream, no_log=True)
+
+        result = self.client.post("/api/v1/external/beanstalk", self.fixture_jsondata('beanstalk', action),
+                                  content_type="application/json",
+                                  HTTP_AUTHORIZATION=self.http_auth(email, api_key))
+        self.assert_json_success(result)
+
+        # Check the correct message was sent
+        msg = Message.objects.filter().order_by('-id')[0]
+        self.assertEqual(msg.sender.user.email, email)
+        self.assertEqual(get_display_recipient(msg.recipient), 'commits')
+
+        return msg
+
+    def test_git_single(self):
+        msg = self.send_beanstalk_message('git_singlecommit')
+        self.assertEqual(msg.subject, "work-test")
+        self.assertEqual(msg.content, """Leo Franchi [pushed](http://lfranchi-svn.beanstalkapp.com/work-test) to branch master
+
+* [e50508d](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/e50508df): add some stuff
+""")
+
+    def test_git_multiple(self):
+        msg = self.send_beanstalk_message('git_multiple')
+        self.assertEqual(msg.subject, "work-test")
+        self.assertEqual(msg.content, """Leo Franchi [pushed](http://lfranchi-svn.beanstalkapp.com/work-test) to branch master
+
+* [edf529c](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/edf529c7): Added new file
+* [c2a191b](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/c2a191b9): Filled in new file with some stuff
+* [2009815](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/20098158): More work to fix some bugs
+""")
+
+    def test_svn_addremove(self):
+        msg = self.send_beanstalk_message('svn_addremove')
+        self.assertEqual(msg.subject, "svn r3")
+        self.assertEqual(msg.content, """Leo Franchi pushed [revision 3](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/3):
+
+> Removed a file and added another one!""")
+
+    def test_svn_changefile(self):
+        msg = self.send_beanstalk_message('svn_changefile')
+        self.assertEqual(msg.subject, "svn r2")
+        self.assertEqual(msg.content, """Leo Franchi pushed [revision 2](http://lfranchi-svn.beanstalkapp.com/work-test/changesets/2):
+
+> Added some code""")
+
 class Runner(DjangoTestSuiteRunner):
     option_list = (
         optparse.make_option('--skip-generate',

zephyr/views.py

@@ -1236,12 +1236,43 @@ def get_activity(request):
             'iPhone':  ActivityTable('iPhone',        api_queries)
         }}, context_instance=RequestContext(request))
 
+def build_message_from_gitlog(user_profile, name, ref, commits, before, after, url, pusher):
+    short_ref = re.sub(r'^refs/heads/', '', ref)
+    subject = name
+
+    if re.match(r'^0+$', after):
+        content = "%s deleted branch %s" % (pusher,
+                                            short_ref)
+    elif len(commits) == 0:
+        content = ("%s [force pushed](%s) to branch %s.  Head is now %s"
+                   % (pusher,
+                      url,
+                      short_ref,
+                      after[:7]))
+    else:
+        content = ("%s [pushed](%s) to branch %s\n\n"
+                   % (pusher,
+                      url,
+                      short_ref))
+        num_commits = len(commits)
+        max_commits = 10
+        truncated_commits = commits[:max_commits]
+        for commit in truncated_commits:
+            short_id = commit['id'][:7]
+            (short_commit_msg, _, _) = commit['message'].partition("\n")
+            content += "* [%s](%s): %s\n" % (short_id, commit['url'],
+                                             short_commit_msg)
+        if (num_commits > max_commits):
+            content += ("\n[and %d more commits]"
+                        % (num_commits - max_commits,))
+
+    return (subject, content)
+
 @authenticated_api_view
 @has_request_variables
 def api_github_landing(request, user_profile, event=POST,
                        payload=POST(converter=json_to_dict)):
     # TODO: this should all be moved to an external bot
-
     repository = payload['repository']
 
     # CUSTOMER18 has requested not to get pull request notifications
@@ -1265,39 +1296,17 @@ def api_github_landing(request, user_profile, event=POST,
         if short_ref != 'master' and user_profile.realm.domain in ['customer18.invalid', 'humbughq.com']:
             return json_success()
 
-        subject = repository['name']
+        subject, content = build_message_from_gitlog(user_profile, repository['name'],
-        if re.match(r'^0+$', payload['after']):
+                                                     payload['ref'], payload['commits'],
-            content = "%s deleted branch %s" % (payload['pusher']['name'],
+                                                     payload['before'], payload['after'],
-                                                short_ref)
+                                                     payload['compare'],
-        elif len(payload['commits']) == 0:
+                                                     payload['pusher']['name'])
-            content = ("%s [force pushed](%s) to branch %s.  Head is now %s"
-                       % (payload['pusher']['name'],
-                          payload['compare'],
-                          short_ref,
-                          payload['after'][:7]))
-        else:
-            content = ("%s [pushed](%s) to branch %s\n\n"
-                       % (payload['pusher']['name'],
-                          payload['compare'],
-                          short_ref))
-            num_commits = len(payload['commits'])
-            max_commits = 10
-            truncated_commits = payload['commits'][:max_commits]
-            for commit in truncated_commits:
-                short_id = commit['id'][:7]
-                (short_commit_msg, _, _) = commit['message'].partition("\n")
-                content += "* [%s](%s): %s\n" % (short_id, commit['url'],
-                                                 short_commit_msg)
-            if (num_commits > max_commits):
-                content += ("\n[and %d more commits]"
-                            % (num_commits - max_commits,))
     else:
         # We don't handle other events even though we get notified
         # about them
         return json_success()
 
-    if len(subject) > MAX_SUBJECT_LENGTH:
+    subject = elide_subject(subject)
-        subject = subject[:57].rstrip() + '...'
 
     request.client = get_client("github_bot")
     return send_message_backend(request, user_profile,
@@ -1306,6 +1315,11 @@ def api_github_landing(request, user_profile, event=POST,
                                 forged=False, subject_name=subject,
                                 message_content=content)
 
+def elide_subject(subject):
+    if len(subject) > MAX_SUBJECT_LENGTH:
+        subject = subject[:57].rstrip() + '...'
+    return subject
+
 def api_jira_webhook(request, api_key):
     payload = simplejson.loads(request.body)
 
@@ -1364,14 +1378,43 @@ def api_jira_webhook(request, api_key):
         if comment != '':
             content += "\n> %s" % (comment,)
 
-    if len(subject) > MAX_SUBJECT_LENGTH:
+    subject = elide_subject(subject)
-        subject = subject[:57].rstrip() + '...'
 
     ret = check_send_message(user_profile, get_client("API"), "stream", ["jira"], subject, content)
     if ret is not None:
         return json_error(ret)
     return json_success()
 
+@authenticated_rest_api_view
+def api_beanstalk_webhook(request, user_profile):
+    payload = simplejson.loads(request.body)
+
+    # Beanstalk supports both SVN and git repositories
+    # We distinguish between the two by checking for a
+    # 'uri' key that is only present for git repos
+    git_repo = 'uri' in payload
+    if git_repo:
+        # To get a linkable url,
+        subject, content = build_message_from_gitlog(user_profile, payload['repository']['name'],
+                                                     payload['ref'], payload['commits'],
+                                                     payload['before'], payload['after'],
+                                                     payload['repository']['url'],
+                                                     payload['pusher_name'])
+    else:
+        author = payload.get('author_full_name')
+        url = payload.get('changeset_url')
+        revision = payload.get('revision')
+        (short_commit_msg, _, _) = payload.get('message').partition("\n")
+
+        subject = "svn r%s" % (revision,)
+        content = "%s pushed [revision %s](%s):\n\n> %s" % (author, revision, url, short_commit_msg)
+
+    subject = elide_subject(subject)
+
+    ret = check_send_message(user_profile, get_client("API"), "stream", ["commits"], subject, content)
+    if ret is not None:
+        return json_error(ret)
+    return json_success()
 
 @cache_with_key(lambda user_profile: user_profile.realm_id, timeout=60)
 def get_status_list(requesting_user_profile):