Skip to content

Commit 7bba3a2

Browse files
Merge branch 'main' into qa
2 parents 4108a15 + bdbde85 commit 7bba3a2

8 files changed

Lines changed: 208 additions & 13 deletions

File tree

CHANGELOG.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Changelog
22

3+
## v0.0.7 - 3/1/23
4+
- Added Oauth2ApiClient for oauth2 authenticated calls to our Platform API and Sierra
5+
- Set PostgreSQL connection pool to have a default pool size minimum of 0
6+
37
## v0.0.5 - 2/22/23
48
- Support write queries to PostgreSQL and MySQL databases
59
- Support different return formats when querying PostgreSQL, MySQL, and Redshift databases
@@ -22,4 +26,4 @@
2226

2327
## v0.0.1 - 1/26/23
2428

25-
Initial version. Includes the `avro_encoder`, `kinesis_client`, `mysql_client`, `postgresql_client`, `redshift_client`, and `s3_client` classes as well as the `config_helper`, `kms_helper`, `log_helper`, and `obfuscation_helper` functions.
29+
Initial version. Includes the `avro_encoder`, `kinesis_client`, `mysql_client`, `postgresql_client`, `redshift_client`, and `s3_client` classes as well as the `config_helper`, `kms_helper`, `log_helper`, and `obfuscation_helper` functions.

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ This package contains common Python utility classes and functions.
1010
* Connecting to and querying a MySQL database
1111
* Connecting to and querying a PostgreSQL database using a connection pool
1212
* Connecting to and querying Redshift
13+
* Making requests to the Oauth2 authenticated APIS such as NYPL Platform API and Sierra
1314

1415
## Functions
1516
* Reading a YAML config file and putting the contents in os.environ
@@ -63,4 +64,4 @@ This repo uses the [Main-QA-Production](https://github.com/NYPL/engineering-gene
6364
- Deploy app to production on GitHub and confirm it works
6465

6566
## Deployment
66-
The utils repo is deployed as a PyPI package [here](https://pypi.org/project/nypl-py-utils/) and as a Test PyPI package for QA purposes [here](https://test.pypi.org/project/nypl-py-utils/). In order to be deployed, the version listed in `pyproject.toml` **must be updated**. To deploy to Test PyPI, create a new release in GitHub and tag it `qa-vX.X.X`. The GitHub Actions deploy-qa workflow will then build and publish the package. To deploy to production PyPI, create a release and tag it `production-vX.X.X`.
67+
The utils repo is deployed as a PyPI package [here](https://pypi.org/project/nypl-py-utils/) and as a Test PyPI package for QA purposes [here](https://test.pypi.org/project/nypl-py-utils/). In order to be deployed, the version listed in `pyproject.toml` **must be updated**. To deploy to Test PyPI, create a new release in GitHub and tag it `qa-vX.X.X`. The GitHub Actions deploy-qa workflow will then build and publish the package. To deploy to production PyPI, create a release and tag it `production-vX.X.X`.

pyproject.toml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
44

55
[project]
66
name = "nypl_py_utils"
7-
version = "0.0.6"
7+
version = "0.0.7"
88
authors = [
99
{ name="Aaron Friedman", email="aaronfriedman@nypl.org" },
1010
]
@@ -22,10 +22,12 @@ dependencies = [
2222
"boto3>=1.26.5",
2323
"botocore>=1.29.5",
2424
"mysql-connector-python>=8.0.32",
25+
"oauthlib>=3.2.2",
2526
"psycopg[binary,pool]>=3.1.6",
2627
"PyYAML>=6.0",
2728
"redshift-connector>=2.0.909",
28-
"requests>=2.28.1"
29+
"requests>=2.28.1",
30+
"requests_oauthlib>=1.3.1"
2931
]
3032

3133
[project.optional-dependencies]
@@ -40,4 +42,4 @@ tests = [
4042

4143
[project.urls]
4244
"Homepage" = "https://github.com/NYPL/python-utils"
43-
"Bug Tracker" = "https://github.com/NYPL/python-utils/issues"
45+
"Bug Tracker" = "https://github.com/NYPL/python-utils/issues"

src/nypl_py_utils/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from .classes.kinesis_client import KinesisClient, KinesisClientError # noqa
33
from .classes.kms_client import KmsClient, KmsClientError # noqa
44
from .classes.mysql_client import MySQLClient, MySQLClientError # noqa
5+
from .classes.oauth2_api_client import Oauth2ApiClient # noqa
56
from .classes.postgresql_client import PostgreSQLClient, PostgreSQLClientError # noqa
67
from .classes.redshift_client import RedshiftClient, RedshiftClientError # noqa
7-
from .classes.s3_client import S3Client, S3ClientError # noqa
8+
from .classes.s3_client import S3Client, S3ClientError # noqa
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import os
2+
from oauthlib.oauth2 import BackendApplicationClient, TokenExpiredError
3+
from requests_oauthlib import OAuth2Session
4+
from nypl_py_utils.functions.log_helper import create_log
5+
6+
7+
class Oauth2ApiClient:
8+
"""
9+
Client for interacting with an Oauth2 authenticated API such as NYPL's
10+
Platform API endpoints
11+
"""
12+
13+
def __init__(self, client_id=None, client_secret=None, base_url=None,
14+
token_url=None):
15+
self.client_id = client_id \
16+
or os.environ.get('NYPL_API_CLIENT_ID', None)
17+
self.client_secret = client_secret \
18+
or os.environ.get('NYPL_API_CLIENT_SECRET', None)
19+
self.token_url = token_url \
20+
or os.environ.get('NYPL_API_TOKEN_URL', None)
21+
self.base_url = base_url \
22+
or os.environ.get('NYPL_API_BASE_URL', None)
23+
24+
self.client = None
25+
self.token = None
26+
27+
self.logger = create_log('oauth2_api_client')
28+
29+
def get(self, request_path, **kwargs):
30+
"""
31+
Issue an HTTP GET on the given request_path
32+
"""
33+
return self._do_http_method('GET', request_path, **kwargs)
34+
35+
def post(self, request_path, json, **kwargs):
36+
"""
37+
Issue an HTTP POST on the given request_path with given JSON body
38+
"""
39+
kwargs['json'] = json
40+
return self._do_http_method('POST', request_path, **kwargs)
41+
42+
def patch(self, request_path, json, **kwargs):
43+
"""
44+
Issue an HTTP PATCH on the given request_path with given JSON body
45+
"""
46+
kwargs['json'] = json
47+
return self._do_http_method('PATCH', request_path, **kwargs)
48+
49+
def delete(self, request_path, **kwargs):
50+
"""
51+
Issue an HTTP DELETE on the given request_path
52+
"""
53+
return self._do_http_method('DELETE', request_path, **kwargs)
54+
55+
def _do_http_method(self, method, request_path, **kwargs):
56+
"""
57+
Issue an HTTP method call on on the given request_path
58+
"""
59+
if not self.client:
60+
self._create_oauth_client()
61+
62+
url = f'{self.base_url}/{request_path}'
63+
self.logger.debug(f'{method} {url}')
64+
65+
try:
66+
return self.oauth_client.request(method, url, **kwargs).json()
67+
except TokenExpiredError as e:
68+
self.logger.debug(f'TokenExpiredError encountered: {e}')
69+
self._generate_access_token()
70+
71+
return self._do_http_method(method, request_path, **kwargs)
72+
except TimeoutError as e:
73+
self.logger.error(f'TimeoutError encountered: {e}')
74+
return {}
75+
76+
def _create_oauth_client(self):
77+
"""
78+
Creates an authenticated a OAuth2Session instance for later requests
79+
"""
80+
self._generate_access_token()
81+
self.oauth_client = OAuth2Session(self.client_id, token=self.token)
82+
83+
def _generate_access_token(self):
84+
"""
85+
Fetch and store a fresh token
86+
"""
87+
client = BackendApplicationClient(client_id=self.client_id)
88+
oauth = OAuth2Session(client=client)
89+
self.logger.debug(f'Refreshing token via @{self.token_url}')
90+
self.token = oauth.fetch_token(
91+
token_url=self.token_url,
92+
client_id=self.client_id,
93+
client_secret=self.client_secret
94+
)

src/nypl_py_utils/classes/postgresql_client.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,11 @@ def __init__(self, host, port, db_name, user, password, **kwargs):
1919
'{db_name}').format(user=user, password=password,
2020
host=host, port=port,
2121
db_name=db_name)
22-
self.min_size = kwargs.get('min_size', 1)
23-
self.max_size = kwargs.get('max_size', None)
22+
self.min_size = kwargs.get('min_size', 0)
23+
self.max_size = kwargs.get('max_size', 1)
2424
self.pool = ConnectionPool(
2525
self.conn_info, open=False,
26-
min_size=kwargs.get('min_size', 1),
27-
max_size=kwargs.get('max_size', None))
26+
min_size=self.min_size, max_size=self.max_size)
2827

2928
def connect(self):
3029
"""
@@ -73,7 +72,6 @@ def execute_query(self, query, is_write_query=False, query_params=None,
7372
"""
7473
self.logger.info('Querying {} database'.format(self.db_name))
7574
self.logger.debug('Executing query {}'.format(query))
76-
self.pool.check()
7775
with self.pool.connection() as conn:
7876
try:
7977
conn.row_factory = row_factory

tests/test_oauth2_api_client.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import os
2+
import json
3+
import pytest
4+
from requests_oauthlib import OAuth2Session
5+
6+
from nypl_py_utils import Oauth2ApiClient
7+
# from requests.exceptions import ConnectTimeout
8+
9+
_TOKEN_RESPONSE = {
10+
'access_token': 'super-secret-token',
11+
'expires_in': 3600,
12+
'token_type': 'Bearer',
13+
'scope': ['offline_access', 'openid', 'login:staff', 'admin'],
14+
'id_token': 'super-secret-token',
15+
'expires_at': 1677599823.3180869
16+
}
17+
18+
BASE_URL = 'https://example.com/api/v0.1'
19+
20+
21+
class TestOauth2ApiClient:
22+
23+
@pytest.fixture
24+
def test_instance(self, requests_mock):
25+
token_url = 'https://oauth.example.com/oauth/token'
26+
requests_mock.post(token_url, text=json.dumps(_TOKEN_RESPONSE))
27+
28+
return Oauth2ApiClient(base_url=BASE_URL,
29+
token_url=token_url,
30+
client_id='clientid',
31+
client_secret='clientsecret'
32+
)
33+
34+
def test_uses_env_vars(self):
35+
env = {
36+
'NYPL_API_CLIENT_ID': 'env client id',
37+
'NYPL_API_CLIENT_SECRET': 'env client secret',
38+
'NYPL_API_TOKEN_URL': 'env token url',
39+
'NYPL_API_BASE_URL': 'env base url'
40+
}
41+
for key, value in env.items():
42+
os.environ[key] = value
43+
44+
client = Oauth2ApiClient()
45+
assert client.client_id == 'env client id'
46+
assert client.client_secret == 'env client secret'
47+
assert client.token_url == 'env token url'
48+
assert client.base_url == 'env base url'
49+
50+
for key, value in env.items():
51+
os.environ[key] = ''
52+
53+
def test_generate_access_token(self, test_instance):
54+
test_instance._generate_access_token()
55+
assert test_instance.token['access_token']\
56+
== _TOKEN_RESPONSE['access_token']
57+
58+
def test_create_oauth_client(self, test_instance):
59+
test_instance._create_oauth_client()
60+
assert type(test_instance.oauth_client) is OAuth2Session
61+
62+
def test_do_http_method(self, requests_mock, test_instance):
63+
requests_mock.get(f'{BASE_URL}/foo', json={'foo': 'bar'})
64+
resp = test_instance._do_http_method('GET', 'foo')
65+
assert resp == {'foo': 'bar'}
66+
67+
def test_token_expiration(self, requests_mock, test_instance):
68+
api_get_mock = requests_mock.get(f'{BASE_URL}/foo',
69+
json={'foo': 'bar'})
70+
71+
# Perform first request to auto-authenticate:
72+
resp = test_instance._do_http_method('GET', 'foo')
73+
assert api_get_mock.request_history[0]._request\
74+
.headers['Authorization'] == 'Bearer super-secret-token'
75+
76+
# Emulate token expiration:
77+
test_instance.token['expires_at'] = 0
78+
79+
token_post_mock = requests_mock.post(
80+
test_instance.token_url,
81+
text=json.dumps(_TOKEN_RESPONSE)
82+
)
83+
84+
# Perform second request, which should detect token expiration and
85+
# re-authenticate:
86+
resp = test_instance._do_http_method('GET', 'foo')
87+
88+
assert token_post_mock.called is True
89+
assert api_get_mock.request_history[1]._request\
90+
.headers['Authorization'] == 'Bearer super-secret-token'
91+
assert resp == {'foo': 'bar'}

tests/test_postgresql_client.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import pytest
22

33
from nypl_py_utils import PostgreSQLClient, PostgreSQLClientError
4+
from psycopg import Error
45

56

67
class TestPostgreSQLClient:
@@ -17,7 +18,7 @@ def test_init(self, test_instance):
1718
'postgresql://test_user:test_password@test_host:test_port/' +
1819
'test_db_name')
1920
assert test_instance.pool._opened is False
20-
assert test_instance.pool.min_size == 1
21+
assert test_instance.pool.min_size == 0
2122
assert test_instance.pool.max_size == 1
2223

2324
def test_init_with_kwargs(self):
@@ -35,7 +36,10 @@ def test_connect(self, test_instance):
3536
test_instance.connect()
3637
test_instance.pool.open.assert_called_once_with(wait=True, timeout=300)
3738

38-
def test_connect_with_exception(self):
39+
def test_connect_with_exception(self, mocker):
40+
mocker.patch('psycopg_pool.ConnectionPool.open',
41+
side_effect=Error())
42+
3943
test_instance = PostgreSQLClient(
4044
'test_host', 'test_port', 'test_db_name', 'test_user',
4145
'test_password', timeout=1.0)

0 commit comments

Comments
 (0)