From b7135b64aa689123b9daa3a3fa56eca679a45e13 Mon Sep 17 00:00:00 2001 From: winkidney Date: Tue, 21 Aug 2018 19:08:35 +0800 Subject: [PATCH 1/4] Feature: Add referer field for Pin --- core/api.py | 2 +- core/migrations/0002_pin_referer.py | 19 +++++++++++++++++++ core/models.py | 1 + pinry/static/js/bookmarklet.js | 3 ++- pinry/static/js/pin-form.js | 16 +++++++++++++--- pinry/templates/includes/lightbox.html | 5 +++-- pinry/templates/includes/pin_form.html | 4 ++++ 7 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 core/migrations/0002_pin_referer.py diff --git a/core/api.py b/core/api.py index 40209b3..229e010 100644 --- a/core/api.py +++ b/core/api.py @@ -137,7 +137,7 @@ class PinResource(ModelResource): return super(PinResource, self).save_m2m(bundle) class Meta: - fields = ['id', 'url', 'origin', 'description'] + fields = ['id', 'url', 'origin', 'description', 'referer'] ordering = ['id'] filtering = { 'submitter': ALL_WITH_RELATIONS diff --git a/core/migrations/0002_pin_referer.py b/core/migrations/0002_pin_referer.py new file mode 100644 index 0000000..6da3afe --- /dev/null +++ b/core/migrations/0002_pin_referer.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='pin', + name='referer', + field=models.URLField(null=True), + ), + ] diff --git a/core/models.py b/core/models.py index 7755866..838c412 100644 --- a/core/models.py +++ b/core/models.py @@ -42,6 +42,7 @@ class Pin(models.Model): submitter = models.ForeignKey(User) url = models.URLField(null=True) origin = models.URLField(null=True) + referer = models.URLField(null=True) description = models.TextField(blank=True, null=True) image = models.ForeignKey(Image, related_name='pin') published = models.DateTimeField(auto_now_add=True) diff --git a/pinry/static/js/bookmarklet.js b/pinry/static/js/bookmarklet.js index bb1df29..ee46d87 100644 --- a/pinry/static/js/bookmarklet.js +++ b/pinry/static/js/bookmarklet.js @@ -92,7 +92,8 @@ border: '1px solid #555' }); image.onclick = function() { - var popUrl = getFormUrl()+encodeURIComponent(imageUrl); + var popUrl = getFormUrl() + encodeURIComponent(imageUrl); + popUrl = popUrl + '&referer=' + encodeURIComponent(window.location); window.open(popUrl); closePinry(); }; diff --git a/pinry/static/js/pin-form.js b/pinry/static/js/pin-form.js index 74436d8..60c89b9 100644 --- a/pinry/static/js/pin-form.js +++ b/pinry/static/js/pin-form.js @@ -17,6 +17,7 @@ $(window).load(function() { return { submitter: currentUser, url: $('#pin-form-image-url').val(), + referer: $('#pin-form-referer').val(), description: $('#pin-form-description').val(), tags: cleanTags($('#pin-form-tags').val()) } @@ -26,6 +27,7 @@ $(window).load(function() { var context = {pins: [{ submitter: currentUser, image: {thumbnail: {image: $('#pin-form-image-url').val()}}, + referer: $('#pin-form-referer').val(), description: $('#pin-form-description').val(), tags: cleanTags($('#pin-form-tags').val()) }]}, @@ -59,9 +61,14 @@ $(window).load(function() { function createPinForm(editPinId) { $('body').append(renderTemplate('#pin-form-template', '')); var modal = $('#pin-form'), - formFields = [$('#pin-form-image-url'), $('#pin-form-description'), - $('#pin-form-tags')], - pinFromUrl = getUrlParameter('pin-image-url'); + formFields = [ + $('#pin-form-image-url'), + $('#pin-form-referer'), + $('#pin-form-description'), + $('#pin-form-tags') + ], + pinFromUrl = getUrlParameter('pin-image-url'), + pinFromReferer = getUrlParameter('referer'); // If editable grab existing data if (editPinId) { var promise = getPinData(editPinId); @@ -69,6 +76,7 @@ $(window).load(function() { editedPin = data; $('#pin-form-image-url').val(editedPin.image.thumbnail.image); $('#pin-form-image-url').parent().hide(); + $('#pin-form-referer').parent().hide(); $('#pin-form-image-upload').parent().hide(); $('#pin-form-description').val(editedPin.description); $('#pin-form-tags').val(editedPin.tags); @@ -113,6 +121,7 @@ $(window).load(function() { if (pinFromUrl) { $('#pin-form-image-upload').parent().css('display', 'none'); $('#pin-form-image-url').val(pinFromUrl); + $('#pin-form-referer').val(pinFromReferer); $('.navbar').css('display', 'none'); modal.css({ 'margin-top': -35, @@ -155,6 +164,7 @@ $(window).load(function() { } else { var data = { submitter: '/api/v1/user/'+currentUser.id+'/', + referer: $('#pin-form-referer').val(), description: $('#pin-form-description').val(), tags: cleanTags($('#pin-form-tags').val()) }; diff --git a/pinry/templates/includes/lightbox.html b/pinry/templates/includes/lightbox.html index ba7dd1e..dafdbec 100644 --- a/pinry/templates/includes/lightbox.html +++ b/pinry/templates/includes/lightbox.html @@ -24,8 +24,9 @@ {{/if}}
- Original URL - Pin URL + Referer + Original URL + Pin URL
diff --git a/pinry/templates/includes/pin_form.html b/pinry/templates/includes/pin_form.html index 21aaa7d..2e750de 100644 --- a/pinry/templates/includes/pin_form.html +++ b/pinry/templates/includes/pin_form.html @@ -17,6 +17,10 @@
+
+ + +
From 5ed61913a102667f5c33c5b91403c30f318dae3c Mon Sep 17 00:00:00 2001 From: winkidney Date: Tue, 21 Aug 2018 20:06:46 +0800 Subject: [PATCH 2/4] Feature: Add UA and referer to fetch pictures from more sites --- core/api.py | 5 ++++- core/models.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/core/api.py b/core/api.py index 229e010..c1361f7 100644 --- a/core/api.py +++ b/core/api.py @@ -103,7 +103,10 @@ class PinResource(ModelResource): def hydrate_image(self, bundle): url = bundle.data.get('url', None) if url: - image = Image.objects.create_for_url(url) + image = Image.objects.create_for_url( + url, + referer=bundle.data.get('referer', None), + ) bundle.data['image'] = '/api/v1/image/{}/'.format(image.pk) return bundle diff --git a/core/models.py b/core/models.py index 838c412..766a598 100644 --- a/core/models.py +++ b/core/models.py @@ -14,11 +14,20 @@ from users.models import User class ImageManager(models.Manager): + _default_ua = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 5.1) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/48.0.2564.82 Safari/537.36', + } + # FIXME: Move this into an asynchronous task - def create_for_url(self, url): + def create_for_url(self, url, referer=None): file_name = url.split("/")[-1].split('#')[0].split('?')[0] buf = BytesIO() - response = requests.get(url) + headers = dict(self._default_ua) + if referer is not None: + headers["Referer"] = referer + response = requests.get(url, headers=headers) buf.write(response.content) obj = InMemoryUploadedFile(buf, 'image', file_name, None, buf.tell(), None) From 696c51458acfeb85642507e2b54c65c071b75104 Mon Sep 17 00:00:00 2001 From: winkidney Date: Tue, 21 Aug 2018 08:05:46 -0700 Subject: [PATCH 3/4] Fix: Fix test-failure for request-test --- users/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/users/tests.py b/users/tests.py index 0e924a1..892a6cc 100644 --- a/users/tests.py +++ b/users/tests.py @@ -9,7 +9,7 @@ from core.models import Image, Pin from .models import User -def mock_requests_get(url): +def mock_requests_get(url, headers=None): response = mock.Mock(content=open('logo.png', 'rb').read()) return response From fadb89880488f5d5b4822657f9ece54a48f8648a Mon Sep 17 00:00:00 2001 From: winkidney Date: Tue, 21 Aug 2018 08:14:17 -0700 Subject: [PATCH 4/4] Fix: Add test for extra referer field --- core/tests/api.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/tests/api.py b/core/tests/api.py index 305783a..da00394 100644 --- a/core/tests/api.py +++ b/core/tests/api.py @@ -67,9 +67,11 @@ class PinResourceTest(ResourceTestCase): @mock.patch('requests.get', mock_requests_get) def test_post_create_url(self): url = 'http://testserver/mocked/logo.png' + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/{}/'.format(self.user.pk), 'url': url, + 'referer': referer, 'description': 'That\'s an Apple!' } response = self.api_client.post('/api/v1/pin/', data=post_data) @@ -89,9 +91,11 @@ class PinResourceTest(ResourceTestCase): @mock.patch('requests.get', mock_requests_get) def test_post_create_url_with_empty_tags(self): url = 'http://testserver/mocked/logo.png' + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/{}/'.format(self.user.pk), 'url': url, + 'referer': referer, 'description': 'That\'s an Apple!', 'tags': [] } @@ -105,9 +109,11 @@ class PinResourceTest(ResourceTestCase): @mock.patch('requests.get', mock_requests_get) def test_post_create_url_unauthorized(self): url = 'http://testserver/mocked/logo.png' + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/2/', 'url': url, + 'referer': referer, 'description': 'That\'s an Apple!', 'tags': [] } @@ -119,9 +125,11 @@ class PinResourceTest(ResourceTestCase): @mock.patch('requests.get', mock_requests_get) def test_post_create_url_with_empty_origin(self): url = 'http://testserver/mocked/logo.png' + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/{}/'.format(self.user.pk), 'url': url, + 'referer': referer, 'description': 'That\'s an Apple!', 'origin': None } @@ -135,9 +143,11 @@ class PinResourceTest(ResourceTestCase): def test_post_create_url_with_origin(self): origin = 'http://testserver/mocked/' url = origin + 'logo.png' + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/{}/'.format(self.user.pk), 'url': url, + 'referer': referer, 'description': 'That\'s an Apple!', 'origin': origin } @@ -149,8 +159,10 @@ class PinResourceTest(ResourceTestCase): def test_post_create_obj(self): image = ImageFactory() + referer = 'http://testserver/' post_data = { 'submitter': '/api/v1/user/{}/'.format(self.user.pk), + 'referer': referer, 'image': '/api/v1/image/{}/'.format(image.pk), 'description': 'That\'s something else (probably a CC logo)!', 'tags': ['random', 'tags'], @@ -228,6 +240,7 @@ class PinResourceTest(ResourceTestCase): pin = PinFactory(**{ 'submitter': self.user, 'image': image, + 'referer': 'http://testserver/mocked/', 'url': 'http://testserver/mocked/logo.png', 'description': u'Mocked Description', 'origin': None