LCORE-1392: Bundle LS providers in LCS image

.tekton/lightspeed-stack-pull-request.yaml

@@ -44,6 +44,10 @@ spec:
            "type": "rpm",
            "path": "."
          },
+         {
+           "type": "generic",
+           "path": "."
+         },
          {
            "type": "pip",
            "path": ".",
@@ -54,7 +58,7 @@ spec:
            ],
            "requirements_build_files": ["requirements-build.txt"],
            "binary": {
-             "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-cloud-resource-manager,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,starlette,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin",
+             "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-cloud-resource-manager,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,starlette,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin",
              "os": "linux",
              "arch": "x86_64,aarch64",
              "py_version": 312

.tekton/lightspeed-stack-push.yaml

@@ -36,6 +36,10 @@ spec:
           "type": "rpm",
           "path": "."
         },
+        {
+          "type": "generic",
+          "path": "."
+        },
         {
           "type": "pip",
           "path": ".",
@@ -46,7 +50,7 @@ spec:
           ],
           "requirements_build_files": ["requirements-build.txt"],
           "binary": {
-            "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-cloud-resource-manager,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,starlette,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin",
+            "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-cloud-resource-manager,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,starlette,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin",
             "os": "linux",
             "arch": "x86_64,aarch64",
             "py_version": 312

Containerfile

@@ -34,6 +34,33 @@ RUN pip3.12 install "uv>=0.8.15"
 COPY ${LSC_SOURCE_DIR}/src ./src
 COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.*.txt ./
 
+# lightspeed-providers:
+# Fully hermetic — uses prefetched artifact or pinned commit from GitHub
+ARG LIGHTSPEED_PROVIDERS_COMMIT=9e073aaaa43a8a5bac38a3bbddbe6cf24842847b
+RUN set -eux; \
+    ZIP_PATH="/tmp/lightspeed-providers.zip"; \
+    EXTRACT_DIR="/tmp/providers"; \
+    \
+    # Use hermetic pre-fetched artifact if available, otherwise download pinned commit
+    if [ -f "/cachi2/output/deps/generic/lightspeed-providers.zip" ]; then \
+        cp "/cachi2/output/deps/generic/lightspeed-providers.zip" "${ZIP_PATH}"; \
+    else \
+        curl -fL "https://github.com/lightspeed-core/lightspeed-providers/archive/${LIGHTSPEED_PROVIDERS_COMMIT}.zip" -o "${ZIP_PATH}"; \
+    fi; \
+    \
+    # Extract zip (stdlib zipfile — no unzip RPM; works on minimal Konflux builders)
+    mkdir -p "${EXTRACT_DIR}"; \
+    export ZIP_PATH EXTRACT_DIR; \
+    ROOT_DIR="$(python3.12 -c 'import os, zipfile; z=zipfile.ZipFile(os.environ["ZIP_PATH"]); print(z.namelist()[0].split("/")[0])')"; \
+    python3.12 -c 'import os, zipfile; zipfile.ZipFile(os.environ["ZIP_PATH"]).extractall(os.environ["EXTRACT_DIR"])'; \
+    \
+    # Move relevant directories into container filesystem
+    mv "${EXTRACT_DIR}/${ROOT_DIR}/lightspeed_stack_providers" /app-root/; \
+    mv "${EXTRACT_DIR}/${ROOT_DIR}/resources/external_providers" /app-root/providers.d; \
+    \
+    # Cleanup
+    rm -rf "${ZIP_PATH}" "${EXTRACT_DIR}"
+
 # Bundle additional dependencies for library mode.
 # Source cachi2 environment for hermetic builds if available, otherwise use normal installation
 # cachi2.env has these env vars:
@@ -99,6 +126,10 @@ RUN mkdir -p /opt/app-root/src/.cache/huggingface && \
 # Add executables from .venv to system PATH
 ENV PATH="/app-root/.venv/bin:$PATH"
 
+# Library mode: Llama Stack expects external provider configs under a path named providers.d (hardcoded).
+# We place them at /app-root/providers.d. YAMLs there reference lightspeed_stack_providers.*, so that package must be on PYTHONPATH.
+ENV PYTHONPATH="/app-root"
+
 # Run the application
 EXPOSE 8080
 ENTRYPOINT ["python3.12", "src/lightspeed_stack.py"]

Makefile

@@ -121,6 +121,9 @@ konflux-requirements:	## Generate hermetic requirements.*.txt file for konflux b
 konflux-rpm-lock:	## Generate rpm.lock.yaml file for konflux build
 	./scripts/generate-rpm-lock.sh
 
+konflux-artifacts-lock: ## Regenerate artifacts.lock.yaml file for konflux build
+	./scripts/generate-artifacts-lock.sh
+
 help: ## Show this help screen
 	@echo 'Usage: make <OPTIONS> ... <TARGETS>'
 	@echo ''

artifacts.lock.yaml

@@ -0,0 +1,8 @@
+# Regenerated by: make konflux-artifacts-lock (from lightspeed-providers.revision)
+metadata:
+  version: "1.0"
+
+artifacts:
+  - download_url: https://github.com/lightspeed-core/lightspeed-providers/archive/9e073aaaa43a8a5bac38a3bbddbe6cf24842847b.zip
+    checksum: sha256:52cd105c351c3645b0b0fbc7c06df5ca200274b843f82f97dfbbb8918aac3a11
+    filename: lightspeed-providers.zip

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
     # Used for Bearer token extraction
     "starlette>=0.47.1",
     # Used by JWK token auth handler
-    "aiohttp>=3.12.14",
+    "aiohttp>=3.13.3",
     "authlib>=1.6.0",
     # Used for A2A protocol support
     "a2a-sdk>=0.3.4,<0.4.0",
@@ -58,7 +58,7 @@ dependencies = [
     "jsonpath-ng>=1.6.1",
     "psycopg2-binary>=2.9.10",
     "litellm>=1.75.5.post1",
-    "urllib3==2.6.3",
+    "urllib3>=2.6.3",
     # Used for agent card configuration
     "PyYAML>=6.0.0",
     # Used for rag faiss support
@@ -188,6 +188,13 @@ llslibdev = [
     "psutil>=7.0.0",
     # Azure
     "azure-identity>=1.21.0",
+    # Lightspeed providers
+    "httpx>=0.27.0",
+    "pydantic>=2.10.6",
+    "numpy>=1.24.0",
+    "mcp>=1.23.0",
+    "protobuf>=6.33.5",
+    "filelock>=3.20.3",
 ]
 
 build = [

requirements-build.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.13
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pybuild-deps compile --output-file=requirements-build.txt requirements.source.txt
@@ -18,10 +18,12 @@ flit-core==3.12.0
     #   wheel
 hatch-fancy-pypi-readme==25.1.0
     # via
+    #   attrs
     #   llama-stack-client
     #   openai
 hatch-vcs==0.5.0
     # via
+    #   attrs
     #   chardet
     #   filelock
 hatchling==1.26.3
@@ -32,6 +34,7 @@ hatchling==1.26.3
 hatchling==1.29.0
     # via
     #   a2a-sdk
+    #   attrs
     #   chardet
     #   filelock
     #   hatch-fancy-pypi-readme