Merge pull request #479 from UiPath/feat/cli-pull

feat: CLI add pull command

Author: radu-mocanu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.6"
+version = "2.1.7"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

src/uipath/_cli/__init__.py

@@ -10,6 +10,7 @@
 from .cli_new import new as new  # type: ignore
 from .cli_pack import pack as pack  # type: ignore
 from .cli_publish import publish as publish  # type: ignore
+from .cli_pull import pull as pull  # type: ignore
 from .cli_push import push as push  # type: ignore
 from .cli_run import run as run  # type: ignore
 
@@ -65,3 +66,4 @@ def cli(lv: bool, v: bool) -> None:
 cli.add_command(auth)
 cli.add_command(invoke)
 cli.add_command(push)
+cli.add_command(pull)

src/uipath/_cli/_utils/_common.py

@@ -1,5 +1,6 @@
 import os
 from typing import Optional
+from urllib.parse import urlparse
 
 import click
 
@@ -29,7 +30,7 @@ def environment_options(function):
     return function
 
 
-def get_env_vars(spinner: Optional[Spinner] = None) -> list[str | None]:
+def get_env_vars(spinner: Optional[Spinner] = None) -> list[str]:
     base_url = os.environ.get("UIPATH_URL")
     token = os.environ.get("UIPATH_ACCESS_TOKEN")
 
@@ -42,7 +43,8 @@ def get_env_vars(spinner: Optional[Spinner] = None) -> list[str | None]:
         click.echo("UIPATH_URL, UIPATH_ACCESS_TOKEN")
         click.get_current_context().exit(1)
 
-    return [base_url, token]
+    # at this step we know for sure that both base_url and token exist. type checking can be disabled
+    return [base_url, token]  # type: ignore
 
 
 def serialize_object(obj):
@@ -69,3 +71,18 @@ def serialize_object(obj):
     # Return primitive types as is
     else:
         return obj
+
+
+def get_org_scoped_url(base_url: str) -> str:
+    """Get organization scoped URL from base URL.
+
+    Args:
+        base_url: The base URL to scope
+
+    Returns:
+        str: The organization scoped URL
+    """
+    parsed = urlparse(base_url)
+    org_name, *_ = parsed.path.strip("/").split("/")
+    org_scoped_url = f"{parsed.scheme}://{parsed.netloc}/{org_name}"
+    return org_scoped_url

src/uipath/_cli/_utils/_studio_project.py

@@ -1,5 +1,6 @@
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
+import httpx
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
@@ -136,3 +137,62 @@ def convert_folder_type(cls, v: Union[str, int, None]) -> Optional[str]:
         if isinstance(v, int):
             return str(v)
         return v
+
+
+def get_project_structure(
+    project_id: str,
+    base_url: str,
+    token: str,
+    client: httpx.Client,
+) -> ProjectStructure:
+    """Retrieve the project's file structure from UiPath Cloud.
+
+    Makes an API call to fetch the complete file structure of a project,
+    including all files and folders. The response is validated against
+    the ProjectStructure model.
+
+    Args:
+        project_id: The ID of the project
+        base_url: The base URL for the API
+        token: Authentication token
+        client: HTTP client to use for requests
+
+    Returns:
+        ProjectStructure: The complete project structure
+
+    Raises:
+        httpx.HTTPError: If the API request fails
+    """
+    headers = {"Authorization": f"Bearer {token}"}
+    url = (
+        f"{base_url}/studio_/backend/api/Project/{project_id}/FileOperations/Structure"
+    )
+
+    response = client.get(url, headers=headers)
+    response.raise_for_status()
+    return ProjectStructure.model_validate(response.json())
+
+
+def download_file(base_url: str, file_id: str, client: httpx.Client, token: str) -> Any:
+    file_url = f"{base_url}/File/{file_id}"
+    response = client.get(file_url, headers={"Authorization": f"Bearer {token}"})
+    response.raise_for_status()
+    return response
+
+
+def get_folder_by_name(
+    structure: ProjectStructure, folder_name: str
+) -> Optional[ProjectFolder]:
+    """Get a folder from the project structure by name.
+
+    Args:
+        structure: The project structure
+        folder_name: Name of the folder to find
+
+    Returns:
+        Optional[ProjectFolder]: The found folder or None
+    """
+    for folder in structure.folders:
+        if folder.name == folder_name:
+            return folder
+    return None

src/uipath/_cli/cli_pack.py

@@ -276,7 +276,9 @@ def display_project_info(config):
 
 
 @click.command()
-@click.argument("root", type=str, default="./")
+@click.argument(
+    "root", type=click.Path(exists=True, file_okay=False, dir_okay=True), default="."
+)
 @click.option(
     "--nolock",
     is_flag=True,

src/uipath/_cli/cli_pull.py

@@ -0,0 +1,215 @@
+# type: ignore
+"""CLI command for pulling remote project files from UiPath StudioWeb solution.
+
+This module provides functionality to pull remote project files from a UiPath StudioWeb solution.
+It handles:
+- File downloads from source_code and evals folders
+- Maintaining folder structure locally
+- File comparison using hashes
+- Interactive confirmation for overwriting files
+"""
+
+# type: ignore
+import hashlib
+import json
+import os
+from typing import Dict, Set
+
+import click
+import httpx
+from dotenv import load_dotenv
+
+from .._utils._ssl_context import get_httpx_client_kwargs
+from ..telemetry import track
+from ._utils._common import get_env_vars, get_org_scoped_url
+from ._utils._console import ConsoleLogger
+from ._utils._studio_project import (
+    ProjectFile,
+    ProjectFolder,
+    download_file,
+    get_folder_by_name,
+    get_project_structure,
+)
+
+console = ConsoleLogger()
+load_dotenv(override=True)
+
+
+def compute_normalized_hash(content: str) -> str:
+    """Compute hash of normalized content.
+
+    Args:
+        content: Content to hash
+
+    Returns:
+        str: SHA256 hash of the normalized content
+    """
+    try:
+        # Try to parse as JSON to handle formatting
+        json_content = json.loads(content)
+        normalized = json.dumps(json_content, indent=2)
+    except json.JSONDecodeError:
+        # Not JSON, normalize line endings
+        normalized = content.replace("\r\n", "\n").replace("\r", "\n")
+
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+
+def collect_files_from_folder(
+    folder: ProjectFolder, base_path: str, files_dict: Dict[str, ProjectFile]
+) -> None:
+    """Recursively collect all files from a folder and its subfolders.
+
+    Args:
+        folder: The folder to collect files from
+        base_path: Base path for file paths
+        files_dict: Dictionary to store collected files
+    """
+    # Add files from current folder
+    for file in folder.files:
+        file_path = os.path.join(base_path, file.name)
+        files_dict[file_path] = file
+
+    # Recursively process subfolders
+    for subfolder in folder.folders:
+        subfolder_path = os.path.join(base_path, subfolder.name)
+        collect_files_from_folder(subfolder, subfolder_path, files_dict)
+
+
+def download_folder_files(
+    folder: ProjectFolder,
+    base_path: str,
+    base_url: str,
+    token: str,
+    client: httpx.Client,
+    processed_files: Set[str],
+) -> None:
+    """Download files from a folder recursively.
+
+    Args:
+        folder: The folder to download files from
+        base_path: Base path for local file storage
+        base_url: Base URL for API requests
+        token: Authentication token
+        client: HTTP client
+        processed_files: Set to track processed files
+    """
+    files_dict: Dict[str, ProjectFile] = {}
+    collect_files_from_folder(folder, "", files_dict)
+
+    for file_path, remote_file in files_dict.items():
+        local_path = os.path.join(base_path, file_path)
+        local_dir = os.path.dirname(local_path)
+
+        # Create directory if it doesn't exist
+        if not os.path.exists(local_dir):
+            os.makedirs(local_dir)
+
+        # Download remote file
+        response = download_file(base_url, remote_file.id, client, token)
+        remote_content = response.read().decode("utf-8")
+        remote_hash = compute_normalized_hash(remote_content)
+
+        if os.path.exists(local_path):
+            # Read and hash local file
+            with open(local_path, "r", encoding="utf-8") as f:
+                local_content = f.read()
+                local_hash = compute_normalized_hash(local_content)
+
+            # Compare hashes
+            if local_hash != remote_hash:
+                styled_path = click.style(str(file_path), fg="cyan")
+                console.warning(f"File {styled_path}" + " differs from remote version.")
+                response = click.prompt("Do you want to override it? (y/n)", type=str)
+                if response.lower() == "y":
+                    with open(local_path, "w", encoding="utf-8", newline="\n") as f:
+                        f.write(remote_content)
+                    console.success(f"Updated {click.style(str(file_path), fg='cyan')}")
+                else:
+                    console.info(f"Skipped {click.style(str(file_path), fg='cyan')}")
+            else:
+                console.info(
+                    f"File {click.style(str(file_path), fg='cyan')} is up to date"
+                )
+        else:
+            # File doesn't exist locally, create it
+            with open(local_path, "w", encoding="utf-8", newline="\n") as f:
+                f.write(remote_content)
+            console.success(f"Downloaded {click.style(str(file_path), fg='cyan')}")
+
+        processed_files.add(file_path)
+
+
+@click.command()
+@click.argument(
+    "root", type=click.Path(exists=True, file_okay=False, dir_okay=True), default="."
+)
+@track
+def pull(root: str) -> None:
+    """Pull remote project files from Studio Web Project.
+
+    This command pulls the remote project files from a UiPath Studio Web project.
+    It downloads files from the source_code and evals folders, maintaining the
+    folder structure locally. Files are compared using hashes before overwriting,
+    and user confirmation is required for differing files.
+
+    Args:
+        root: The root directory to pull files into
+
+    Environment Variables:
+        UIPATH_PROJECT_ID: Required. The ID of the UiPath Studio Web project
+
+    Example:
+        $ uipath pull
+        $ uipath pull /path/to/project
+    """
+    if not os.getenv("UIPATH_PROJECT_ID", False):
+        console.error("UIPATH_PROJECT_ID environment variable not found.")
+
+    [base_url, token] = get_env_vars()
+    base_api_url = f"{get_org_scoped_url(base_url)}/studio_/backend/api/Project/{os.getenv('UIPATH_PROJECT_ID')}/FileOperations"
+
+    with console.spinner("Pulling UiPath project files..."):
+        try:
+            with httpx.Client(**get_httpx_client_kwargs()) as client:
+                # Get project structure
+                structure = get_project_structure(
+                    os.getenv("UIPATH_PROJECT_ID"),  # type: ignore
+                    get_org_scoped_url(base_url),
+                    token,
+                    client,
+                )
+
+                processed_files: Set[str] = set()
+
+                # Process source_code folder
+                source_code_folder = get_folder_by_name(structure, "source_code")
+                if source_code_folder:
+                    download_folder_files(
+                        source_code_folder,
+                        root,
+                        base_api_url,
+                        token,
+                        client,
+                        processed_files,
+                    )
+                else:
+                    console.warning("No source_code folder found in remote project")
+
+                # Process evals folder
+                evals_folder = get_folder_by_name(structure, "evals")
+                if evals_folder:
+                    evals_path = os.path.join(root, "evals")
+                    download_folder_files(
+                        evals_folder,
+                        evals_path,
+                        base_api_url,
+                        token,
+                        client,
+                        processed_files,
+                    )
+                else:
+                    console.warning("No evals folder found in remote project")
+
+        except Exception as e:
+            console.error(f"Failed to pull UiPath project: {str(e)}")

src/uipath/_cli/cli_push.py

@@ -492,7 +492,9 @@ def upload_source_files_to_project(
 
 
 @click.command()
-@click.argument("root", type=str, default="./")
+@click.argument(
+    "root", type=click.Path(exists=True, file_okay=False, dir_okay=True), default="."
+)
 @click.option(
     "--nolock",
     is_flag=True,