From b575daf430946167baf688e486503d69fe495c5a Mon Sep 17 00:00:00 2001 From: kd Date: Thu, 2 Jul 2020 14:54:55 -0400 Subject: updated with black and pyflakes run --- README.md | 3 ++ main.py | 30 ++++++++++--------- msgraph.py | 93 ++++++++++++++++++++++++++------------------------------- slack_parser.py | 32 ++++++++++++-------- test.py | 41 +++++++++++++------------ 5 files changed, 103 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 6a70d6a..6d4081e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # SlackToTeams +Code style: Black [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +Code linter: Pyflakes [![Code linter: Pyflakes](https://badge.fury.io/py/pyflakes.svg)](https://github.com/PyCQA/pyflakes) +--- ## What it does Migrate Slack messages into Teams. The information that gets retained for each message: - the (user) sender of the message. diff --git a/main.py b/main.py index 9518267..0f7eb3c 100644 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ from msgraph import MSGraphApi logging.basicConfig(level=logging.INFO) LIMIT_THROUGHPUT = False -if __name__ == '__main__': +if __name__ == "__main__": # run the tests first suite = unittest.TestLoader().loadTestsFromModule(test) @@ -24,7 +24,7 @@ if __name__ == '__main__': config = json.load(open(sys.argv[1])) - #msapi = MSGraphApi(config).getAuth() # "app flow" + # msapi = MSGraphApi(config).getAuth() # "app flow" msapi = MSGraphApi(config, flow="obo").getAuth() if msapi is not None: @@ -34,19 +34,21 @@ if __name__ == '__main__': teamId = msapi.getTeamByName(config["teamname"]) channelId = msapi.getChatByNameAndTeamId(teamId, config["channelname"]) - #graph_data = msapi.getUsers().json() - #graph_data = msapi.getChatByTeamAndId(teamId, channelId).json() - #graph_data = msapi.writeChatMessageByTeamAndId( + # graph_data = msapi.getUsers().json() + # graph_data = msapi.getChatByTeamAndId(teamId, channelId).json() + # graph_data = msapi.writeChatMessageByTeamAndId( # teamId=teamId, # channelId=channelId, # payload={ "body" : { "content" : "hello world" } } - #).json() + # ).json() - #print("Graph API call result: ") - #print(json.dumps(graph_data, indent=2)) - #sys.exit(0) + # print("Graph API call result: ") + # print(json.dumps(graph_data, indent=2)) + # sys.exit(0) - sp = SlackParser("resources/channels/devops", "resources/users.json", ["text", "ts", "name"]).buildUsersLibrary() + sp = SlackParser( + "resources/channels/devops", "resources/users.json", ["text", "ts", "name"] + ).buildUsersLibrary() if sp.usersLib is {}: logging.error("Failed to read users. Shutting down with code 1.") sys.exit(1) @@ -62,9 +64,7 @@ if __name__ == '__main__': if reduced_message is not None: teams_mssg = msapi.buildTeamsMessage(reduced_message) graph_data = msapi.writeChatMessageByTeamAndId( - teamId=teamId, - channelId=channelId, - payload=teams_mssg + teamId=teamId, channelId=channelId, payload=teams_mssg ) if graph_data.status_code != 201: logging.error("failed to write to Teams") @@ -82,5 +82,7 @@ if __name__ == '__main__': time.sleep(1) else: - logging.error("Failed to Authenticate with Ms Graph. Shutting down with code 1.") + logging.error( + "Failed to Authenticate with Ms Graph. Shutting down with code 1." + ) sys.exit(1) diff --git a/msgraph.py b/msgraph.py index 0ed77bc..00d5b6c 100644 --- a/msgraph.py +++ b/msgraph.py @@ -3,6 +3,7 @@ import json import requests import msal + class MSGraphApi(object): """Docstring for MSGraphApi. """ @@ -21,8 +22,7 @@ class MSGraphApi(object): self.flow = flow if self.flow == "obo": app = msal.PublicClientApplication( - self.config["client_id"], - authority=self.config["authority"] + self.config["client_id"], authority=self.config["authority"] ) else: # no flow was given, assume app flow @@ -46,8 +46,8 @@ class MSGraphApi(object): # Firstly, looks up a token from cache if self.flow == "obo": result = self.app.acquire_token_silent( - self.config["scope"], - account=self.app.get_accounts(self.config["username"]) + self.config["scope"], + account=self.app.get_accounts(self.config["username"]), ) else: # Since we are looking for token for the current app, NOT for an end user, @@ -55,13 +55,15 @@ class MSGraphApi(object): result = self.app.acquire_token_silent(self.config["scope"], account=None) if not result: - logging.info("No suitable token exists in cache. Let's get a new one from AAD.") + logging.info( + "No suitable token exists in cache. Let's get a new one from AAD." + ) if self.flow == "obo": result = self.app.acquire_token_by_username_password( - self.config["username"], - self.config["password"], - scopes=self.config["scope"] + self.config["username"], + self.config["password"], + scopes=self.config["scope"], ) else: result = self.app.acquire_token_for_client(scopes=self.config["scope"]) @@ -73,7 +75,9 @@ class MSGraphApi(object): else: logging.error(result.get("error")) logging.error(result.get("error_description")) - logging.error(result.get("correlation_id")) # may need this when reporting a bug + logging.error( + result.get("correlation_id") + ) # may need this when reporting a bug return None def setAccessToken(self, tok): @@ -93,12 +97,9 @@ class MSGraphApi(object): """ endpoint = "https://graph.microsoft.com/v1.0/users" - headers = { "Authorization": "Bearer " + self.access_token } + headers = {"Authorization": "Bearer " + self.access_token} - graph_data = requests.get( - endpoint, - headers=headers - ) + graph_data = requests.get(endpoint, headers=headers) return graph_data def getMyTeams(self): @@ -109,15 +110,11 @@ class MSGraphApi(object): """ endpoint = "https://graph.microsoft.com/beta/me/joinedTeams" - headers = { "Authorization": "Bearer " + self.access_token } + headers = {"Authorization": "Bearer " + self.access_token} - graph_data = requests.get( - endpoint, - headers=headers - ) + graph_data = requests.get(endpoint, headers=headers) return graph_data - def getChatsByTeam(self, teamId): """Docstring for getChatsByTeam. Get all chats in given team id. @@ -127,12 +124,9 @@ class MSGraphApi(object): """ endpoint = "https://graph.microsoft.com/beta/teams/%s/channels/" % (teamId) - headers = { "Authorization": "Bearer " + self.access_token } + headers = {"Authorization": "Bearer " + self.access_token} - graph_data = requests.get( - endpoint, - headers=headers - ) + graph_data = requests.get(endpoint, headers=headers) return graph_data def getChatByTeamAndId(self, teamId, channelId): @@ -144,13 +138,13 @@ class MSGraphApi(object): :returns: respose object containing ms response """ - endpoint = "https://graph.microsoft.com/beta/teams/%s/channels/%s" % (teamId, channelId) - headers = { "Authorization": "Bearer " + self.access_token } - - graph_data = requests.get( - endpoint, - headers=headers + endpoint = "https://graph.microsoft.com/beta/teams/%s/channels/%s" % ( + teamId, + channelId, ) + headers = {"Authorization": "Bearer " + self.access_token} + + graph_data = requests.get(endpoint, headers=headers) return graph_data def writeChatMessageByTeamAndId(self, teamId, channelId, payload): @@ -163,22 +157,21 @@ class MSGraphApi(object): """ # other pathways - #userId = "8a18e679-3603-4e10-a5e4-c75ad362fde0" - #endpoint = "https://graph.microsoft.com/beta/users/%s/chats/%s/messages" % (userId, channelId) - #endpoint = "https://graph.microsoft.com/beta/chats/%s/messages" % (channelId) + # userId = "8a18e679-3603-4e10-a5e4-c75ad362fde0" + # endpoint = "https://graph.microsoft.com/beta/users/%s/chats/%s/messages" % (userId, channelId) + # endpoint = "https://graph.microsoft.com/beta/chats/%s/messages" % (channelId) # ms recommended pathway - endpoint = "https://graph.microsoft.com/beta/teams/%s/channels/%s/messages/" % (teamId, channelId) + endpoint = "https://graph.microsoft.com/beta/teams/%s/channels/%s/messages/" % ( + teamId, + channelId, + ) headers = { "Authorization": "Bearer " + self.access_token, - "Content-Type": "application/json" + "Content-Type": "application/json", } - graph_data = requests.post( - endpoint, - headers=headers, - data=json.dumps(payload) - ) + graph_data = requests.post(endpoint, headers=headers, data=json.dumps(payload)) return graph_data def getTeamByName(self, teamName): @@ -234,19 +227,19 @@ class MSGraphApi(object): # schema teams_mssg = { "messageType": "message", - "body": { - "content": "", - "contentType": "html" - } + "body": {"content": "", "contentType": "html"}, } # convert ``` snippets into block quotes ie. #
some code
rich_html_content = MSGraphApi.insertCodeBlocksInMessage(customMessage["text"]) - html_content = "" + \ - "

From: %s %s

" % (customMessage["name"], customMessage["ts"]) + \ - "

%s

" % (rich_html_content) + \ - "" + html_content = ( + "" + + "

From: %s %s

" + % (customMessage["name"], customMessage["ts"]) + + "

%s

" % (rich_html_content) + + "" + ) split_content = html_content.split("\n") if len(split_content) > 1: @@ -257,7 +250,7 @@ class MSGraphApi(object): # TODO: sanitize string - #teams_mssg["body"]["content"] = html_content + # teams_mssg["body"]["content"] = html_content teams_mssg["body"]["content"] = split_content_str return teams_mssg diff --git a/slack_parser.py b/slack_parser.py index 8756992..d99e7c2 100644 --- a/slack_parser.py +++ b/slack_parser.py @@ -2,17 +2,14 @@ import os import json import re import time -from jsonpath_ng import jsonpath, parse +from jsonpath_ng import parse # jsonpath match will match in order of appearence. # This mapping defines which position in the RESULT of match to look for which key. # so. eg. at 0th position (of the match) will be the key "text", etc. -key_orderings = { - "text": 0, - "ts": 1, - "name": 2 -} +key_orderings = {"text": 0, "ts": 1, "name": 2} + class SlackParser(object): @@ -39,7 +36,7 @@ class SlackParser(object): :returns: a sorted list of json files names for the channel """ - files = sorted(os.listdir( self.messagesDir )) + files = sorted(os.listdir(self.messagesDir)) return files def getUsers(self): @@ -54,7 +51,6 @@ class SlackParser(object): content = json.load(open(path)) return content - def buildUsersLibrary(self): """Docstring for buildUsersLibrary Build a Users dict that keeps only usersid to name mappings. @@ -107,7 +103,7 @@ class SlackParser(object): jsonpath_expression = parse("$.." + str(self.keys)) # search slackMessage for each key, and add all matches (incl. dups.) in-order match = jsonpath_expression.find(slackMessage) - #print(match[0]) + # print(match[0]) for key in self.keys: # TODO: validation for matches possible "index out of range" here. data[key] = match[key_orderings[key]].value @@ -115,11 +111,17 @@ class SlackParser(object): userIds = self.getUserIdsFromMessage(data["text"]) if userIds != []: # find corresponding usernames from library - userNamesinMessageDict = self.getUserNamesFromIds(self.usersLib, userIds) + userNamesinMessageDict = self.getUserNamesFromIds( + self.usersLib, userIds + ) # inject usernames in the text replace userid - data["text"] = self.replaceMessageUserIdsWithNames(userNamesinMessageDict, data["text"]) + data["text"] = self.replaceMessageUserIdsWithNames( + userNamesinMessageDict, data["text"] + ) # convert ts to a human readble time - data["ts"] = time.strftime( "%H:%M %b %d, %Y", time.localtime(float(data["ts"])) ) + data["ts"] = time.strftime( + "%H:%M %b %d, %Y", time.localtime(float(data["ts"])) + ) return data @@ -165,5 +167,9 @@ class SlackParser(object): """ for userId in messageUsersDict.keys(): - text = re.sub( "<@{}>".format(userId), "@{}".format(messageUsersDict[userId]), text ) + text = re.sub( + "<@{}>".format(userId), + "@{}".format(messageUsersDict[userId]), + text, + ) return text diff --git a/test.py b/test.py index 81daa0b..0c5ad0d 100644 --- a/test.py +++ b/test.py @@ -3,10 +3,12 @@ import unittest from msgraph import MSGraphApi from slack_parser import SlackParser + class Test(unittest.TestCase): """ The basic class that inherits unittest.TestCase """ + msapi = MSGraphApi() sp = SlackParser() @@ -15,19 +17,19 @@ class Test(unittest.TestCase): input_string = "sdfsdf" expected = "sdfsdf" actual = self.msapi.insertCodeBlocksInMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_1_insert_block_quotes(self): input_string = "``` CODE1 ```" expected = "
CODE1
" actual = self.msapi.insertCodeBlocksInMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_2_insert_block_quotes(self): input_string = " there and ``` CODE 1 ``` asdfasdf ``` CODE 2 ``` and then again ``` CODE 3 ``` end." expected = " there and
CODE 1
asdfasdf
CODE 2
and then again
CODE 3
end." actual = self.msapi.insertCodeBlocksInMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_3_insert_long_block_quotes(self): input_string = """<@U5F956SJH> a long running query on the master from job01.....job-consumer-2 logs showing\n```>1|ERROR|2020-03-17 15:50:00,018-0400|org.springframework.jms.listener.DefaultMessageListenerContainer#0-3|GenerateEmailNotificationTemplate:106|Unexpected error in processing. Abending.\norg.mybatis.spring.MyBatisSystemException: nested exception is org.apache.ibatis.exceptions.PersistenceException:\n### Error querying database. Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications\n### Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications\n at org.mybatis.spring.MyBatisExceptionTranslator.translateExceptionIfPossible(MyBatisExceptionTranslator.java:77)\n at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:446)\n at com.sun.proxy.$Proxy49.selectList(Unknown Source)\n at org.mybatis.spring.SqlSessionTemplate.selectList(SqlSessionTemplate.java:230)\n at com.acuityads.persistence.dao.SelectQuery.selectList(SelectQuery.java:74)```\n @@ -36,7 +38,7 @@ class Test(unittest.TestCase): """ actual = self.msapi.insertCodeBlocksInMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_4_insert_block_quotes(self): input_string = """<@U5F956SJH> a long running query on the master from job01.....job-consumer-2 logs showing

```>1|ERROR|2020-03-17 15:50:00,018-0400|org.springframework.jms.listener.DefaultMessageListenerContainer#0-3|GenerateEmailNotificationTemplate:106|Unexpected error in processing. Abending.

org.mybatis.spring.MyBatisSystemException: nested exception is org.apache.ibatis.exceptions.PersistenceException:

### Error querying database. Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications

### Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications

at org.mybatis.spring.MyBatisExceptionTranslator.translateExceptionIfPossible(MyBatisExceptionTranslator.java:77)

at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:446)

at com.sun.proxy.$Proxy49.selectList(Unknown Source)

at org.mybatis.spring.SqlSessionTemplate.selectList(SqlSessionTemplate.java:230)

at com.acuityads.persistence.dao.SelectQuery.selectList(SelectQuery.java:74)```

@@ -44,53 +46,54 @@ class Test(unittest.TestCase): expected = """<@U5F956SJH> a long running query on the master from job01.....job-consumer-2 logs showing

>1|ERROR|2020-03-17 15:50:00,018-0400|org.springframework.jms.listener.DefaultMessageListenerContainer#0-3|GenerateEmailNotificationTemplate:106|Unexpected error in processing. Abending.

org.mybatis.spring.MyBatisSystemException: nested exception is org.apache.ibatis.exceptions.PersistenceException:

### Error querying database. Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications

### Cause: java.lang.IllegalArgumentException: Mapped Statements collection does not contain value for com.acuityads.notification.dao.NotificationMapper.findNotifications

at org.mybatis.spring.MyBatisExceptionTranslator.translateExceptionIfPossible(MyBatisExceptionTranslator.java:77)

at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:446)

at com.sun.proxy.$Proxy49.selectList(Unknown Source)

at org.mybatis.spring.SqlSessionTemplate.selectList(SqlSessionTemplate.java:230)

at com.acuityads.persistence.dao.SelectQuery.selectList(SelectQuery.java:74)

""" actual = self.msapi.insertCodeBlocksInMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_5_parse_user_ids(self): input_string = "a long history of thisg" expected = [] actual = self.sp.getUserIdsFromMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_6_parse_user_ids(self): input_string = "<@U32498ksjl> a long history of thisg" expected = ["U32498ksjl"] actual = self.sp.getUserIdsFromMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_7_parse_user_ids(self): input_string = " what

<@U32498ksjl> a long history

of thisg <@kumar12>" expected = ["U32498ksjl", "kumar12"] actual = self.sp.getUserIdsFromMessage(input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_8_get_user_name_from_user_ids(self): - usersDict = { "U32498ksjl" : "kd1" , "UBLAH" : "kd2" } + usersDict = {"U32498ksjl": "kd1", "UBLAH": "kd2"} test_input = ["U32498ksjl"] - expected = { "U32498ksjl" : "kd1" } + expected = {"U32498ksjl": "kd1"} actual = self.sp.getUserNamesFromIds(usersDict, test_input) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_9_get_user_name_from_user_ids(self): - usersDict = { "U32498ksjl" : "kd1" , "UBLAH" : "kd2" } + usersDict = {"U32498ksjl": "kd1", "UBLAH": "kd2"} test_input = ["U32498ksjl", "UBLAH"] - expected = { "U32498ksjl" : "kd1", "UBLAH" : "kd2" } + expected = {"U32498ksjl": "kd1", "UBLAH": "kd2"} actual = self.sp.getUserNamesFromIds(usersDict, test_input) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_10_replace_userids_with_names(self): - usersDict = { "U32498ksjl" : "kd1" } + usersDict = {"U32498ksjl": "kd1"} input_string = "<@U32498ksjl> a long history of thisg" expected = "@kd1 a long history of thisg" actual = self.sp.replaceMessageUserIdsWithNames(usersDict, input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) def test_11_replace_userids_with_names(self): - usersDict = { "U32498ksjl" : "kd1" , "UBLAH" : "kd2" } + usersDict = {"U32498ksjl": "kd1", "UBLAH": "kd2"} input_string = "<@U32498ksjl> a long history of thisg from <@UBLAH> and this <@UBLAH> again." expected = "@kd1 a long history of thisg from @kd2 and this @kd2 again." actual = self.sp.replaceMessageUserIdsWithNames(usersDict, input_string) - self.assertEqual( expected, actual ) + self.assertEqual(expected, actual) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() -- cgit v1.2.3