Feat/type hints

.github/workflows/build.yaml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: [ master ]
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -37,4 +40,32 @@ jobs:
     - name: Run build tests
       run: |
         python -m pip install pytest
-        python -m pytest tests/ci/build-test/
+        python -m pytest tests/ci/build-test/
+
+  typing:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Install dependencies
+      run: |
+        python -V
+        python -m pip install --upgrade pip setuptools
+        python -m pip install mypy types-requests types-six
+        python -m pip install protobuf lz4
+        python -m pip install .
+
+    - name: Run stubtest
+      run: |
+        python tests/ci/run_stubtest.py
+
+    - name: Check stub signatures match source
+      run: |
+        python tests/ci/check_stub_signatures.py

.gitignore

@@ -17,3 +17,4 @@ cscope.in.out
 cscope.out
 cscope.po.out
 tags
+.venv-stubtest/

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include aliyun *.pyi py.typed
+recursive-include aliyun/log *.pyi py.typed

aliyun/log/__init__.pyi

@@ -0,0 +1,56 @@
+from .logclient import LogClient as LogClient
+from .logexception import LogException as LogException
+from .gethistogramsrequest import GetHistogramsRequest as GetHistogramsRequest
+from .getlogsrequest import GetLogsRequest as GetLogsRequest, GetProjectLogsRequest as GetProjectLogsRequest
+from .index_config import IndexConfig as IndexConfig, IndexKeyConfig as IndexKeyConfig, IndexLineConfig as IndexLineConfig
+from .listtopicsrequest import ListTopicsRequest as ListTopicsRequest
+from .listlogstoresrequest import ListLogstoresRequest as ListLogstoresRequest
+from .logtail_config_detail import PluginConfigDetail as PluginConfigDetail, SeperatorFileConfigDetail as SeperatorFileConfigDetail, SimpleFileConfigDetail as SimpleFileConfigDetail, FullRegFileConfigDetail as FullRegFileConfigDetail, JsonFileConfigDetail as JsonFileConfigDetail, ApsaraFileConfigDetail as ApsaraFileConfigDetail, SyslogConfigDetail as SyslogConfigDetail, LogtailConfigGenerator as LogtailConfigGenerator, CommonRegLogConfigDetail as CommonRegLogConfigDetail
+from .logtail_pipeline_config_detail import LogtailPipelineConfigDetail as LogtailPipelineConfigDetail
+from .logtail_pipeline_config_response import CreateLogtailPipelineConfigResponse as CreateLogtailPipelineConfigResponse, DeleteLogtailPipelineConfigResponse as DeleteLogtailPipelineConfigResponse, GetLogtailPipelineConfigResponse as GetLogtailPipelineConfigResponse, UpdateLogtailPipelineConfigResponse as UpdateLogtailPipelineConfigResponse, ListLogtailPipelineConfigResponse as ListLogtailPipelineConfigResponse
+from .machine_group_detail import MachineGroupDetail as MachineGroupDetail
+from .putlogsrequest import PutLogsRequest as PutLogsRequest
+from .shipper_config import ShipperTask as ShipperTask, OssShipperConfig as OssShipperConfig, OdpsShipperConfig as OdpsShipperConfig
+from .version import __version__ as __version__
+from .logitem import LogItem as LogItem
+from .consumer_group_request import CreateConsumerGroupRequest as CreateConsumerGroupRequest, ConsumerGroupGetCheckPointRequest as ConsumerGroupGetCheckPointRequest, ConsumerGroupHeartBeatRequest as ConsumerGroupHeartBeatRequest, ConsumerGroupUpdateCheckPointRequest as ConsumerGroupUpdateCheckPointRequest
+from .external_store_config import ExternalStoreConfig as ExternalStoreConfig, ExternalStoreConfigBase as ExternalStoreConfigBase, ExternalStoreCsvConfig as ExternalStoreCsvConfig, ExternalStoreOssConfig as ExternalStoreOssConfig
+
+from .consumer_group_response import ConsumerGroupEntity as ConsumerGroupEntity, ConsumerGroupCheckPointResponse as ConsumerGroupCheckPointResponse, ConsumerGroupHeartBeatResponse as ConsumerGroupHeartBeatResponse, ConsumerGroupUpdateCheckPointResponse as ConsumerGroupUpdateCheckPointResponse, CreateConsumerGroupResponse as CreateConsumerGroupResponse, DeleteConsumerGroupResponse as DeleteConsumerGroupResponse, ListConsumerGroupResponse as ListConsumerGroupResponse, UpdateConsumerGroupResponse as UpdateConsumerGroupResponse
+from .cursor_response import GetCursorResponse as GetCursorResponse
+from .cursor_time_response import GetCursorTimeResponse as GetCursorTimeResponse
+from .gethistogramsresponse import GetHistogramsResponse as GetHistogramsResponse
+from .getlogsresponse import GetLogsResponse as GetLogsResponse
+from .histogram import Histogram as Histogram
+from .queriedlog import QueriedLog as QueriedLog
+from .index_config_response import CreateIndexResponse as CreateIndexResponse, UpdateIndexResponse as UpdateIndexResponse, DeleteIndexResponse as DeleteIndexResponse, GetIndexResponse as GetIndexResponse
+from .listlogstoresresponse import ListLogstoresResponse as ListLogstoresResponse
+from .listtopicsresponse import ListTopicsResponse as ListTopicsResponse
+from .logresponse import LogResponse as LogResponse
+from .logtail_config_response import CreateLogtailConfigResponse as CreateLogtailConfigResponse, DeleteLogtailConfigResponse as DeleteLogtailConfigResponse, GetLogtailConfigResponse as GetLogtailConfigResponse, UpdateLogtailConfigResponse as UpdateLogtailConfigResponse, ListLogtailConfigResponse as ListLogtailConfigResponse
+from .machinegroup_response import CreateMachineGroupResponse as CreateMachineGroupResponse, DeleteMachineGroupResponse as DeleteMachineGroupResponse, GetMachineGroupResponse as GetMachineGroupResponse, UpdateMachineGroupResponse as UpdateMachineGroupResponse, ListMachineGroupResponse as ListMachineGroupResponse, ListMachinesResponse as ListMachinesResponse, ApplyConfigToMachineGroupResponse as ApplyConfigToMachineGroupResponse, RemoveConfigToMachineGroupResponse as RemoveConfigToMachineGroupResponse, GetMachineGroupAppliedConfigResponse as GetMachineGroupAppliedConfigResponse, GetConfigAppliedMachineGroupsResponse as GetConfigAppliedMachineGroupsResponse
+from .project_response import CreateProjectResponse as CreateProjectResponse, UpdateProjectResponse as UpdateProjectResponse, DeleteProjectResponse as DeleteProjectResponse, GetProjectResponse as GetProjectResponse, ListProjectResponse as ListProjectResponse, GetProjectTagsResponse as GetProjectTagsResponse
+from .pulllog_response import PullLogResponse as PullLogResponse
+from .shard_response import ListShardResponse as ListShardResponse, DeleteShardResponse as DeleteShardResponse
+from .shipper_response import CreateShipperResponse as CreateShipperResponse, UpdateShipperResponse as UpdateShipperResponse, DeleteShipperResponse as DeleteShipperResponse, GetShipperConfigResponse as GetShipperConfigResponse, ListShipperResponse as ListShipperResponse, GetShipperTasksResponse as GetShipperTasksResponse, RetryShipperTasksResponse as RetryShipperTasksResponse
+from .common_response import CreateEntityResponse as CreateEntityResponse, UpdateEntityResponse as UpdateEntityResponse, DeleteEntityResponse as DeleteEntityResponse, GetEntityResponse as GetEntityResponse, ListEntityResponse as ListEntityResponse
+from .external_store_config_response import CreateExternalStoreResponse as CreateExternalStoreResponse, DeleteExternalStoreResponse as DeleteExternalStoreResponse, GetExternalStoreResponse as GetExternalStoreResponse, UpdateExternalStoreResponse as UpdateExternalStoreResponse, ListExternalStoreResponse as ListExternalStoreResponse
+from .proto import LogGroupRaw as LogGroup
+from .rebuild_index_response import CreateRebuildIndexResponse as CreateRebuildIndexResponse, GetRebuildIndexResponse as GetRebuildIndexResponse
+from .deletelogsrequest import DeleteLogsRequest as DeleteLogsRequest
+from .deletelogssresponse import DeleteLogsResponse as DeleteLogsResponse
+from .getdeletelogsstatusrequest import GetDeleteLogsStatusRequest as GetDeleteLogsStatusRequest
+from .getdeletelogsstatusresponse import GetDeleteLogsStatusResponse as GetDeleteLogsStatusResponse
+from .listdeletelogsstasksrequest import ListDeleteLogsTasksRequest as ListDeleteLogsTasksRequest
+from .listdeletelogsstasksresponse import ListDeleteLogsTasksResponse as ListDeleteLogsTasksResponse
+from .logger_hanlder import SimpleLogHandler as SimpleLogHandler, QueuedLogHandler as QueuedLogHandler, LogFields as LogFields, UwsgiQueuedLogHandler as UwsgiQueuedLogHandler
+from .metering_mode_response import GetLogStoreMeteringModeResponse as GetLogStoreMeteringModeResponse, \
+    GetMetricStoreMeteringModeResponse as GetMetricStoreMeteringModeResponse, \
+    UpdateLogStoreMeteringModeResponse as UpdateLogStoreMeteringModeResponse, UpdateMetricStoreMeteringModeResponse as UpdateMetricStoreMeteringModeResponse
+from .multimodal_config_response import GetLogStoreMultimodalConfigurationResponse as GetLogStoreMultimodalConfigurationResponse, \
+    PutLogStoreMultimodalConfigurationResponse as PutLogStoreMultimodalConfigurationResponse
+from .object_response import PutObjectResponse as PutObjectResponse, GetObjectResponse as GetObjectResponse
+
+from .store_view import StoreView as StoreView, StoreViewStore as StoreViewStore
+from .store_view_response import CreateStoreViewResponse as CreateStoreViewResponse, UpdateStoreViewResponse as UpdateStoreViewResponse, DeleteStoreViewResponse as DeleteStoreViewResponse, ListStoreViewsResponse as ListStoreViewsResponse, GetStoreViewResponse as GetStoreViewResponse
+from .submit_async_sql_request import SubmitAsyncSqlRequest as SubmitAsyncSqlRequest

aliyun/log/consumer/__init__.pyi

@@ -0,0 +1,3 @@
+from .config import CursorPosition as CursorPosition, LogHubConfig as LogHubConfig
+from .tasks import ConsumerJsonProcessorBase as ConsumerJsonProcessorBase, ConsumerProcessorAdaptor as ConsumerProcessorAdaptor, ConsumerProcessorBase as ConsumerProcessorBase
+from .worker import ConsumerWorker as ConsumerWorker

aliyun/log/consumer/config.pyi

@@ -0,0 +1,40 @@
+from typing import Any, Callable, Optional
+from enum import Enum
+
+class CursorPosition(Enum):
+    BEGIN_CURSOR = 'BEGIN_CURSOR'
+    END_CURSOR = 'END_CURSOR'
+    SPECIAL_TIMER_CURSOR = 'SPECIAL_TIMER_CURSOR'
+
+class ConsumerStatus(Enum):
+    INITIALIZING = 'INITIALIZING'
+    PROCESSING = 'PROCESSING'
+    SHUTTING_DOWN = 'SHUTTING_DOWN'
+    SHUTDOWN_COMPLETE = 'SHUTDOWN_COMPLETE'
+
+class LogHubConfig:
+    endpoint: str
+    accessKeyId: str
+    accessKey: str
+    project: str
+    logstore: str
+    consumer_group_name: str
+    consumer_name: str
+    cursor_position: CursorPosition
+    heartbeat_interval: int
+    data_fetch_interval: int
+    in_order: bool
+    cursor_start_time: Any
+    securityToken: Optional[str]
+    max_fetch_log_group_size: int
+    worker_pool_size: int
+    shared_executor: Any
+    consumer_group_time_out: int
+    cursor_end_time: Any
+    credentials_refresher: Optional[Callable[..., Any]]
+    auth_version: str
+    region: str
+    query: Optional[str]
+    accept_compress_type: Optional[str]
+    processor: Optional[str]
+    def __init__(self, endpoint: str, access_key_id: str, access_key: str, project: str, logstore: str, consumer_group_name: str, consumer_name: str, cursor_position: Optional[CursorPosition] = ..., heartbeat_interval: Optional[int] = ..., data_fetch_interval: Optional[int] = ..., in_order: bool = False, cursor_start_time: Any = ..., security_token: Optional[str] = ..., max_fetch_log_group_size: Optional[int] = ..., worker_pool_size: Optional[int] = ..., shared_executor: Any = ..., cursor_end_time: Any = ..., credentials_refresher: Optional[Callable[..., Any]] = ..., auth_version: str = ..., region: str = '', query: Optional[str] = ..., accept_compress_type: Optional[str] = ..., processor: Optional[str] = ...) -> None: ...

aliyun/log/consumer/tasks.py

@@ -214,8 +214,7 @@ def consumer_fetch_task(loghub_client_adapter, preprocessor, shard_id, cursor, m
 
     for retry_times in range(3):
         try:
-            # type: PullLogResponse
-            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query, processor=consume_processor)
+            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query, processor=consume_processor)  # type: PullLogResponse
             data = preprocessor(response)
             next_cursor = response.get_next_cursor()
             raw_size = response.get_raw_size()

aliyun/log/consumer/tasks.pyi

@@ -0,0 +1,23 @@
+import abc
+from typing import Any
+
+class ConsumerProcessorBase(object, metaclass=abc.ABCMeta):
+    shard_id: int
+    last_check_time: float
+    checkpoint_timeout: int
+    def __init__(self) -> None: ...
+    def save_checkpoint(self, tracker: Any, force: bool = False) -> None: ...
+    def initialize(self, shard: int) -> None: ...
+    @abc.abstractmethod
+    def process(self, log_groups: Any, check_point_tracker: Any) -> Any: ...
+    def shutdown(self, check_point_tracker: Any) -> None: ...
+
+class ConsumerJsonProcessorBase(ConsumerProcessorBase, metaclass=abc.ABCMeta):
+    def __init__(self) -> None: ...
+    @abc.abstractmethod
+    def process(self, flattern_json_list: Any, check_point_tracker: Any) -> Any: ...
+
+class ConsumerProcessorAdaptor(ConsumerProcessorBase):
+    func: Any
+    def __init__(self, func: Any) -> None: ...
+    def process(self, log_groups: Any, check_point_tracker: Any) -> None: ...

aliyun/log/consumer/worker.pyi

@@ -0,0 +1,28 @@
+import logging
+from typing import Any, Dict, MutableMapping, Optional, Sequence, Tuple
+from threading import Thread
+from .config import LogHubConfig as LogHubConfig
+
+class ConsumerWorkerLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> Tuple[Any, MutableMapping[str, Any]]: ...
+
+class ConsumerWorker(Thread):
+    make_processor: Any
+    process_args: Sequence[Any]
+    process_kwargs: Dict[str, Any]
+    option: LogHubConfig
+    consumer_client: Any
+    shut_down_flag: bool
+    logger: logging.LoggerAdapter
+    shard_consumers: Dict[Any, Any]
+    last_owned_consumer_finish_time: float
+    heart_beat: Any
+    own_executor: bool
+    def __init__(self, make_processor: Any, consumer_option: LogHubConfig, args: Optional[Sequence[Any]] = ..., kwargs: Optional[Dict[str, Any]] = ...) -> None: ...
+    @property
+    def executor(self) -> Any: ...
+    def run(self) -> None: ...
+    def start(self, join: bool = False) -> None: ...
+    def shutdown_and_wait(self) -> None: ...
+    def clean_shard_consumer(self, owned_shards: Sequence[Any]) -> None: ...
+    def shutdown(self) -> None: ...

aliyun/log/credentials.pyi

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from typing import Optional
+
+class Credentials:
+    access_key_id: str
+    access_key_secret: str
+    security_token: Optional[str]
+    def __init__(self, access_key_id: str, access_key_secret: str, security_token: Optional[str] = ...) -> None: ...
+    def get_access_key_id(self) -> str: ...
+    def get_access_key_secret(self) -> str: ...
+    def get_security_token(self) -> Optional[str]: ...
+
+class CredentialsProvider:
+    def get_credentials(self) -> Optional[Credentials]: ...
+
+class StaticCredentialsProvider(CredentialsProvider):
+    def __init__(self, access_key_id: str = ..., access_key_secret: str = ..., security_token: str = ...) -> None: ...
+    def get_credentials(self) -> Credentials: ...

aliyun/log/cursor_response.pyi

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+from typing import Any, Dict
+
+from .logresponse import LogResponse
+
+class GetCursorResponse(LogResponse):
+    def __init__(self, resp: Dict[str, Any], header: Dict[str, Any]) -> None: ...
+    def get_cursor(self) -> str: ...
+    def log_print(self) -> None: ...

aliyun/log/cursor_time_response.pyi

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+from typing import Any, Dict
+
+from .logresponse import LogResponse
+
+class GetCursorTimeResponse(LogResponse):
+    def __init__(self, resp: Dict[str, Any], header: Dict[str, Any]) -> None: ...
+    def get_cursor_time(self) -> int: ...
+    def log_print(self) -> None: ...

aliyun/log/gethistogramsrequest.pyi

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from typing import Optional, Union
+
+from .logrequest import LogRequest
+from .util import parse_timestamp
+
+class GetHistogramsRequest(LogRequest):
+    def __init__(self, project: Optional[str] = ..., logstore: Optional[str] = ..., fromTime: Optional[Union[int, str]] = ..., toTime: Optional[Union[int, str]] = ..., topic: Optional[str] = ..., query: Optional[str] = ..., accurate_query: bool = ..., from_time_nano_part: int = ..., to_time_nano_part: int = ..., shard_id: int = ...) -> None: ...
+    def get_logstore(self) -> str: ...
+    def set_logstore(self, logstore: str) -> None: ...
+    def get_topic(self) -> str: ...
+    def set_topic(self, topic: str) -> None: ...
+    def get_from(self) -> int: ...
+    def set_from(self, fromTime: int) -> None: ...
+    def get_to(self) -> int: ...
+    def set_to(self, toTime: int) -> None: ...
+    def get_query(self) -> str: ...
+    def set_query(self, query: str) -> None: ...
+    def get_accurate_query(self) -> bool: ...
+    def set_accurate_query(self, accurate_query: bool) -> None: ...
+    def get_from_time_nano_part(self) -> int: ...
+    def set_from_time_nano_part(self, from_time_nano_part: int) -> None: ...
+    def get_to_time_nano_part(self) -> int: ...
+    def set_to_time_nano_part(self, to_time_nano_part: int) -> None: ...
+    def set_shard_id(self, shard_id: int) -> None: ...
+    def get_shard_id(self) -> int: ...

aliyun/log/gethistogramsresponse.pyi

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
+
+from .logresponse import LogResponse
+from .histogram import Histogram
+from .util import Util
+
+class GetHistogramsResponse(LogResponse):
+    def __init__(self, resp: Dict[str, Any], header: Dict[str, Any]) -> None: ...
+    def is_completed(self) -> bool: ...
+    def get_total_count(self) -> int: ...
+    def get_histograms(self) -> List[Histogram]: ...
+    def log_print(self) -> None: ...

aliyun/log/getlogsrequest.pyi

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
+
+from .logrequest import LogRequest
+
+class GetLogsRequest(LogRequest):
+    def __init__(self, project: Optional[str] = ..., logstore: Optional[str] = ..., fromTime: Optional[Union[int, str]] = ..., toTime: Optional[Union[int, str]] = ..., topic: Optional[str] = ..., query: Optional[str] = ..., line: int = ..., offset: int = ..., reverse: bool = ..., power_sql: bool = ..., scan: bool = ..., forward: bool = ..., accurate_query: bool = ..., from_time_nano_part: int = ..., to_time_nano_part: int = ...) -> None: ...
+    def get_logstore(self) -> str: ...
+    def set_logstore(self, logstore: str) -> None: ...
+    def get_topic(self) -> str: ...
+    def set_topic(self, topic: str) -> None: ...
+    def get_from(self) -> int: ...
+    def set_from(self, fromTime: int) -> None: ...
+    def get_to(self) -> int: ...
+    def set_to(self, toTime: int) -> None: ...
+    def get_query(self) -> str: ...
+    def set_query(self, query: str) -> None: ...
+    def get_line(self) -> int: ...
+    def set_line(self, line: int) -> None: ...
+    def get_offset(self) -> int: ...
+    def set_offset(self, offset: int) -> None: ...
+    def get_reverse(self) -> bool: ...
+    def set_reverse(self, reverse: bool) -> None: ...
+    def get_power_sql(self) -> bool: ...
+    def set_power_sql(self, power_sql: bool) -> None: ...
+    def get_scan(self) -> bool: ...
+    def set_scan(self, scan: bool) -> None: ...
+    def get_forward(self) -> bool: ...
+    def set_forward(self, forward: bool) -> None: ...
+    def get_accurate_query(self) -> bool: ...
+    def set_accurate_query(self, accurate_query: bool) -> None: ...
+    def get_from_time_nano_part(self) -> int: ...
+    def set_from_time_nano_part(self, from_time_nano_part: int) -> None: ...
+    def get_to_time_nano_part(self) -> int: ...
+    def set_to_time_nano_part(self, to_time_nano_part: int) -> None: ...
+
+class GetProjectLogsRequest(LogRequest):
+    def __init__(self, project: Optional[str] = ..., query: Optional[str] = ..., power_sql: bool = ...) -> None: ...
+    def get_query(self) -> str: ...
+    def set_query(self, query: str) -> None: ...
+    def get_power_sql(self) -> bool: ...
+    def set_power_sql(self, power_sql: bool) -> None: ...