diff --git a/CHANGES/278.feature b/CHANGES/278.feature new file mode 100644 index 0000000..23b5ea0 --- /dev/null +++ b/CHANGES/278.feature @@ -0,0 +1 @@ +Added pull-through cache feature. diff --git a/pulp_npm/app/models.py b/pulp_npm/app/models.py index 92a2888..9c89ba5 100644 --- a/pulp_npm/app/models.py +++ b/pulp_npm/app/models.py @@ -1,10 +1,3 @@ -""" -Check `Plugin Writer's Guide`_ for more details. - -.. _Plugin Writer's Guide: - http://docs.pulpproject.org/en/3.0/nightly/plugins/plugin-writer/index.html -""" - import json from logging import getLogger @@ -21,7 +14,7 @@ ) from pulpcore.plugin.util import get_domain_pk -from .utils import urlpath_sanitize +from .utils import urlpath_sanitize, extract_package_info logger = getLogger(__name__) @@ -32,16 +25,6 @@ class Package(Content): Define fields you need for your new content type and specify uniqueness constraint to identify unit of this type. - - For example:: - - field1 = models.TextField() - field2 = models.IntegerField() - field3 = models.CharField() - - class Meta: - default_related_name = "%(app_label)s_%(model_name)s" - unique_together = (field1, field2) """ TYPE = "package" @@ -58,6 +41,15 @@ def relative_path(self): """ return f"{self.name}/-/{self.name}-{self.version}.tgz" + @staticmethod + def init_from_artifact_and_relative_path(artifact, relative_path): + name, version = extract_package_info(relative_path) + + if not version: + return Package(name=name) + + return Package(name=name, version=version) + class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("name", "version", "_pulp_domain") @@ -72,6 +64,15 @@ class NpmRemote(Remote): TYPE = "npm" + def get_remote_artifact_url(self, relative_path=None, request=None): + if request and (url := request.query.get("redirect")): + # This is a special case for pull-through caching + return url + return super().get_remote_artifact_url(relative_path, request=request) + + def get_remote_artifact_content_type(self, relative_path=None): + return Package + class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -105,6 +106,9 @@ class Meta: def content_handler(self, path): data = {} + if not self.repository: + return None + repository_version = self.repository_version if not repository_version: repository_version = self.repository.latest_version() diff --git a/pulp_npm/app/serializers.py b/pulp_npm/app/serializers.py index 4ce16ab..b01ded0 100644 --- a/pulp_npm/app/serializers.py +++ b/pulp_npm/app/serializers.py @@ -1,7 +1,8 @@ +from gettext import gettext as _ from rest_framework import serializers from pulpcore.plugin import models as core_models -from pulpcore.plugin import serializers as platform +from pulpcore.plugin import serializers as core_serializers from . import models @@ -12,7 +13,7 @@ # If you want create content through upload, use "SingleArtifactContentUploadSerializer" # If you change this, make sure to do so on "fields" below, also. # Make sure your choice here matches up with the create() method of your viewset. -class PackageSerializer(platform.SingleArtifactContentUploadSerializer): +class PackageSerializer(core_serializers.SingleArtifactContentUploadSerializer): """ A Serializer for Package. @@ -26,7 +27,7 @@ class PackageSerializer(platform.SingleArtifactContentUploadSerializer): field3 = serializers.CharField() class Meta: - fields = platform.SingleArtifactContentSerializer.Meta.fields + ( + fields = core_serializers.SingleArtifactContentSerializer.Meta.fields + ( 'field1', 'field2', 'field3' ) model = models.Package @@ -37,7 +38,7 @@ class Meta: relative_path = serializers.CharField() class Meta: - fields = platform.SingleArtifactContentUploadSerializer.Meta.fields + ( + fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( "name", "version", "relative_path", @@ -45,7 +46,7 @@ class Meta: model = models.Package -class NpmRemoteSerializer(platform.RemoteSerializer): +class NpmRemoteSerializer(core_serializers.RemoteSerializer): """ A Serializer for NpmRemote. @@ -56,9 +57,9 @@ class NpmRemoteSerializer(platform.RemoteSerializer): For example:: class Meta: - validators = platform.RemoteSerializer.Meta.validators + [myValidator1, myValidator2] + validators = core_serializers.RemoteSerializer.Meta.validators + [myValidator1, ...] - By default the 'policy' field in platform.RemoteSerializer only validates the choice + By default the 'policy' field in core_serializers.RemoteSerializer only validates the choice 'immediate'. To add on-demand support for more 'policy' options, e.g. 'streamed' or 'on_demand', re-define the 'policy' option as follows:: @@ -72,11 +73,11 @@ class Meta: ) class Meta: - fields = platform.RemoteSerializer.Meta.fields + fields = core_serializers.RemoteSerializer.Meta.fields model = models.NpmRemote -class NpmRepositorySerializer(platform.RepositorySerializer): +class NpmRepositorySerializer(core_serializers.RepositorySerializer): """ A Serializer for NpmRepository. @@ -87,19 +88,27 @@ class NpmRepositorySerializer(platform.RepositorySerializer): For example:: class Meta: - validators = platform.RepositorySerializer.Meta.validators + [myValidator1, myValidator2] + validators = core_serializers.RepositorySerializer.Meta.validators + [myValidator1, ...] """ class Meta: - fields = platform.RepositorySerializer.Meta.fields + fields = core_serializers.RepositorySerializer.Meta.fields model = models.NpmRepository -class NpmDistributionSerializer(platform.DistributionSerializer): +class NpmDistributionSerializer(core_serializers.DistributionSerializer): """ Serializer for NPM Distributions. """ + remote = core_serializers.DetailRelatedField( + required=False, + help_text=_("Remote that can be used to fetch content when using pull-through caching."), + view_name_pattern=r"remotes(-.*/.*)?-detail", + queryset=core_models.Remote.objects.all(), + allow_null=True, + ) + class Meta: - fields = platform.DistributionSerializer.Meta.fields + fields = core_serializers.DistributionSerializer.Meta.fields + ("remote",) model = models.NpmDistribution diff --git a/pulp_npm/app/utils.py b/pulp_npm/app/utils.py index 6f3dd55..ce4767e 100644 --- a/pulp_npm/app/utils.py +++ b/pulp_npm/app/utils.py @@ -1,3 +1,6 @@ +import re + + def urlpath_sanitize(*args): """ Join an arbitrary number of strings into a /-separated path. @@ -15,3 +18,23 @@ def urlpath_sanitize(*args): if stripped: segments.append(stripped) return "/".join(segments) + + +def extract_package_info(package_string): + """ + Tries to extract the name and the version of a package + from the relative path string. + + Args: + The relative_path string. "package/-/package-version.tgz" + """ + pattern = r"(?P[^/]+)(?:/-/(?P=name)-(?P[^\.]+)\.\w+)?$" + + match = re.match(pattern, package_string) + + if match: + name = match.group("name") + version = match.group("version") + return name, version + else: + return None, None diff --git a/pulp_npm/tests/functional/api/test_download_content.py b/pulp_npm/tests/functional/api/test_download_content.py index 8c9ffc5..9184ed5 100644 --- a/pulp_npm/tests/functional/api/test_download_content.py +++ b/pulp_npm/tests/functional/api/test_download_content.py @@ -9,9 +9,7 @@ from pulpcore.client.pulp_npm import RepositorySyncURL -from pulp_npm.tests.functional.constants import ( - NPM_FIXTURE_URL, -) +from pulp_npm.tests.functional.constants import NPM_FIXTURE_URL @pytest.mark.parallel diff --git a/pulp_npm/tests/functional/api/test_pull_through_caching.py b/pulp_npm/tests/functional/api/test_pull_through_caching.py new file mode 100644 index 0000000..be42174 --- /dev/null +++ b/pulp_npm/tests/functional/api/test_pull_through_caching.py @@ -0,0 +1,17 @@ +import pytest + +from urllib.parse import urljoin, urlsplit + +from pulp_npm.tests.functional.constants import NPM_FIXTURE_URL + + +def test_pull_through_install(npm_bindings, npm_remote_factory, npm_distribution_factory, http_get): + """Test that a pull-through distro can be installed from.""" + remote = npm_remote_factory(url=NPM_FIXTURE_URL) + distro = npm_distribution_factory(remote=remote.pulp_href) + PACKAGE = "commander" + + __import__("ipdb").set_trace() + ye = http_get(f"{distro.base_url}/react") + # content = npm_bindings.ContentPackagesApi.list(name=PACKAGE) + # assert content.count == 1