diff --git a/.circleci/config.yml b/.circleci/config.yml index 70a9e7f08e4..bacfc65e622 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,234 +456,6 @@ promote_common: &promote_common # Job specs ############################################################################## jobs: - pytorch_linux_build: - <<: *pytorch_params - machine: - image: ubuntu-2004:202104-01 - steps: - # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml - - checkout - - calculate_docker_image_tag - - setup_linux_system_environment - - optional_merge_target_branch - - setup_ci_environment - - run: - name: Build - no_output_timeout: "1h" - command: | - set -e - if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then - echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}" - fi - if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" - echo 'USE_TBB=1' >> "${BASH_ENV}" - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}" - fi - echo "Parallel backend flags: "${PARALLEL_FLAGS} - # Pull Docker image and run build - echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG} - time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) - - git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 - - docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace - - export COMMAND='((echo "sudo chown -R jenkins workspace && export JOB_BASE_NAME="$CIRCLE_JOB" && cd workspace && .jenkins/pytorch/build.sh && find ${BUILD_ROOT} -type f -name "*.a" -or -name "*.o" -delete") | docker exec -u jenkins -i "$id" bash) 2>&1' - - echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts - - # Copy dist folder back - docker cp $id:/var/lib/jenkins/workspace/dist /home/circleci/project/. || echo "Dist folder not found" - - # Push intermediate Docker image for next phase to use - if [ -z "${BUILD_ONLY}" ]; then - # Note [Special build images] - # The xla build uses the same docker image as - # pytorch_linux_bionic_py3_6_clang9_build. In the push step, we have to - # distinguish between them so the test can pick up the correct image. - output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1} - if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-xla - elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-libtorch - elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-parallelnative - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_64"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-x86_64 - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v7a"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v7a - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v8a"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v8a - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_32"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-x86_32 - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-vulkan-x86_32"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-vulkan-x86_32 - elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-vulkan - else - export COMMIT_DOCKER_IMAGE=$output_image - fi - docker commit "$id" ${COMMIT_DOCKER_IMAGE} - time docker push ${COMMIT_DOCKER_IMAGE} - fi - - run: - name: upload build & binary data - no_output_timeout: "5m" - command: | - cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) - python3 -mpip install requests && \ - SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ - python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - - store_artifacts: - path: /home/circleci/project/dist - - pytorch_linux_test: - <<: *pytorch_params - machine: - image: ubuntu-2004:202104-01 - steps: - # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml - - checkout - - calculate_docker_image_tag - - setup_linux_system_environment - - setup_ci_environment - - run: - name: Download Docker image - no_output_timeout: "90m" - command: | - set -e - export PYTHONUNBUFFERED=1 - if [[ "${DOCKER_IMAGE}" == *rocm3.9* ]]; then - export DOCKER_TAG="f3d89a32912f62815e4feaeed47e564e887dffd6" - fi - # See Note [Special build images] - output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1} - if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-xla - elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-libtorch - elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-parallelnative - elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-vulkan - else - export COMMIT_DOCKER_IMAGE=$output_image - fi - echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE} - - if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" - echo 'USE_TBB=1' >> "${BASH_ENV}" - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}" - fi - echo "Parallel backend flags: "${PARALLEL_FLAGS} - - time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null - - # TODO: Make this less painful - if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all --shm-size=2g -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - elif [[ ${BUILD_ENVIRONMENT} == *"rocm"* ]]; then - hostname - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=8g --ipc=host --device /dev/kfd --device /dev/dri --group-add video -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - else - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=1g --ipc=host -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - fi - echo "id=${id}" >> "${BASH_ENV}" - - - run: - name: Check for no AVX instruction by default - no_output_timeout: "20m" - command: | - set -e - is_vanilla_build() { - if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-bionic-py3.7-clang9-test" ]; then - return 0 - fi - if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-xenial-py3.7-gcc5.4-test" ]; then - return 0 - fi - return 1 - } - - if is_vanilla_build; then - echo "apt-get update || apt-get install libgnutls30" | docker exec -u root -i "$id" bash - echo "apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash - echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash - else - echo "Skipping for ${BUILD_ENVIRONMENT}" - fi - - run: - name: Test - no_output_timeout: "90m" - command: | - set -e - - cat >docker_commands.sh \<> docker_commands.sh - elif [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then - echo ".jenkins/caffe2/test.sh" >> docker_commands.sh - else - echo ".jenkins/pytorch/test.sh" >> docker_commands.sh - fi - echo "(cat docker_commands.sh | docker exec -u jenkins -i "$id" bash) 2>&1" > command.sh - unbuffer bash command.sh | ts - - - run: - name: Report results - no_output_timeout: "5m" - command: | - set -e - # Retrieving test results should be done as very first step as command never fails - # But is always executed if previous step fails for some reason - echo "Retrieving test reports" - docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!' - docker stats --all --no-stream - - cat >docker_commands.sh \<&1" > command.sh - unbuffer bash command.sh | ts - when: always - - store_test_results: - path: test-reports binary_linux_build: <<: *binary_linux_build_params steps: diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py index 9a6dae00f8b..92b152cb253 100755 --- a/.circleci/generate_config_yml.py +++ b/.circleci/generate_config_yml.py @@ -185,7 +185,6 @@ YAML_SOURCES = [ File("build-parameters/binary-build-params.yml"), File("build-parameters/promote-build-params.yml"), Header("Job specs"), - File("job-specs/pytorch-job-specs.yml"), File("job-specs/binary-job-specs.yml"), File("job-specs/job-specs-custom.yml"), File("job-specs/job-specs-promote.yml"), diff --git a/.circleci/verbatim-sources/job-specs/binary-job-specs.yml b/.circleci/verbatim-sources/job-specs/binary-job-specs.yml index 5dd8dab85c9..581b76c8f94 100644 --- a/.circleci/verbatim-sources/job-specs/binary-job-specs.yml +++ b/.circleci/verbatim-sources/job-specs/binary-job-specs.yml @@ -1,3 +1,4 @@ +jobs: binary_linux_build: <<: *binary_linux_build_params steps: diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml deleted file mode 100644 index 79f879a13f0..00000000000 --- a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml +++ /dev/null @@ -1,229 +0,0 @@ -jobs: - pytorch_linux_build: - <<: *pytorch_params - machine: - image: ubuntu-2004:202104-01 - steps: - # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml - - checkout - - calculate_docker_image_tag - - setup_linux_system_environment - - optional_merge_target_branch - - setup_ci_environment - - run: - name: Build - no_output_timeout: "1h" - command: | - set -e - if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then - echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}" - fi - if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" - echo 'USE_TBB=1' >> "${BASH_ENV}" - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}" - fi - echo "Parallel backend flags: "${PARALLEL_FLAGS} - # Pull Docker image and run build - echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG} - time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) - - git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 - - docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace - - export COMMAND='((echo "sudo chown -R jenkins workspace && export JOB_BASE_NAME="$CIRCLE_JOB" && cd workspace && .jenkins/pytorch/build.sh && find ${BUILD_ROOT} -type f -name "*.a" -or -name "*.o" -delete") | docker exec -u jenkins -i "$id" bash) 2>&1' - - echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts - - # Copy dist folder back - docker cp $id:/var/lib/jenkins/workspace/dist /home/circleci/project/. || echo "Dist folder not found" - - # Push intermediate Docker image for next phase to use - if [ -z "${BUILD_ONLY}" ]; then - # Note [Special build images] - # The xla build uses the same docker image as - # pytorch_linux_bionic_py3_6_clang9_build. In the push step, we have to - # distinguish between them so the test can pick up the correct image. - output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1} - if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-xla - elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-libtorch - elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-parallelnative - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_64"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-x86_64 - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v7a"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v7a - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-arm-v8a"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-arm-v8a - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-x86_32"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-x86_32 - elif [[ ${BUILD_ENVIRONMENT} == *"android-ndk-r19c-vulkan-x86_32"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-android-vulkan-x86_32 - elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-vulkan - else - export COMMIT_DOCKER_IMAGE=$output_image - fi - docker commit "$id" ${COMMIT_DOCKER_IMAGE} - time docker push ${COMMIT_DOCKER_IMAGE} - fi - - run: - name: upload build & binary data - no_output_timeout: "5m" - command: | - cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) - python3 -mpip install requests && \ - SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \ - python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - - store_artifacts: - path: /home/circleci/project/dist - - pytorch_linux_test: - <<: *pytorch_params - machine: - image: ubuntu-2004:202104-01 - steps: - # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml - - checkout - - calculate_docker_image_tag - - setup_linux_system_environment - - setup_ci_environment - - run: - name: Download Docker image - no_output_timeout: "90m" - command: | - set -e - export PYTHONUNBUFFERED=1 - if [[ "${DOCKER_IMAGE}" == *rocm3.9* ]]; then - export DOCKER_TAG="f3d89a32912f62815e4feaeed47e564e887dffd6" - fi - # See Note [Special build images] - output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1} - if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-xla - elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-libtorch - elif [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-paralleltbb - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-parallelnative - elif [[ ${BUILD_ENVIRONMENT} == *"vulkan-linux"* ]]; then - export COMMIT_DOCKER_IMAGE=$output_image-vulkan - else - export COMMIT_DOCKER_IMAGE=$output_image - fi - echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE} - - if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then - echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" - echo 'USE_TBB=1' >> "${BASH_ENV}" - elif [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then - echo 'ATEN_THREADING=NATIVE' >> "${BASH_ENV}" - fi - echo "Parallel backend flags: "${PARALLEL_FLAGS} - - time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null - - # TODO: Make this less painful - if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all --shm-size=2g -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - elif [[ ${BUILD_ENVIRONMENT} == *"rocm"* ]]; then - hostname - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=8g --ipc=host --device /dev/kfd --device /dev/dri --group-add video -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - else - export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=1g --ipc=host -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE}) - fi - echo "id=${id}" >> "${BASH_ENV}" - - - run: - name: Check for no AVX instruction by default - no_output_timeout: "20m" - command: | - set -e - is_vanilla_build() { - if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-bionic-py3.7-clang9-test" ]; then - return 0 - fi - if [ "${BUILD_ENVIRONMENT}" == "pytorch-linux-xenial-py3.7-gcc5.4-test" ]; then - return 0 - fi - return 1 - } - - if is_vanilla_build; then - echo "apt-get update || apt-get install libgnutls30" | docker exec -u root -i "$id" bash - echo "apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash - echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash - else - echo "Skipping for ${BUILD_ENVIRONMENT}" - fi - - run: - name: Test - no_output_timeout: "90m" - command: | - set -e - - cat >docker_commands.sh \<> docker_commands.sh - elif [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then - echo ".jenkins/caffe2/test.sh" >> docker_commands.sh - else - echo ".jenkins/pytorch/test.sh" >> docker_commands.sh - fi - echo "(cat docker_commands.sh | docker exec -u jenkins -i "$id" bash) 2>&1" > command.sh - unbuffer bash command.sh | ts - - - run: - name: Report results - no_output_timeout: "5m" - command: | - set -e - # Retrieving test results should be done as very first step as command never fails - # But is always executed if previous step fails for some reason - echo "Retrieving test reports" - docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!' - docker stats --all --no-stream - - cat >docker_commands.sh \<&1" > command.sh - unbuffer bash command.sh | ts - when: always - - store_test_results: - path: test-reports diff --git a/tools/stats/upload_binary_size_to_scuba.py b/tools/stats/upload_binary_size_to_scuba.py index abd86129c3c..adf1d507686 100644 --- a/tools/stats/upload_binary_size_to_scuba.py +++ b/tools/stats/upload_binary_size_to_scuba.py @@ -118,11 +118,13 @@ def report_android_sizes(file_dir: str) -> None: "pkg_type": "{}/{}/{}".format(android_build_type, arch, lib), "cu_ver": "", # dummy value for derived field `build_name` "py_ver": "", # dummy value for derived field `build_name` - "pr": os.environ.get("CIRCLE_PR_NUMBER"), + "pr": os.environ.get("PR_NUMBER", os.environ.get("CIRCLE_PR_NUMBER")), + # This is the only place where we use directly CIRCLE_BUILD_NUM, everywhere else CIRCLE_* vars + # are used as fallback, there seems to be no direct analogy between circle build number and GHA IDs "build_num": os.environ.get("CIRCLE_BUILD_NUM"), - "sha1": os.environ.get("CIRCLE_SHA1"), - "branch": os.environ.get("CIRCLE_BRANCH"), - "workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID"), + "sha1": os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1")), + "branch": os.environ.get("BRANCH", os.environ.get("CIRCLE_BRANCH")), + "workflow_id": os.environ.get("WORKFLOW_ID", os.environ.get("CIRCLE_WORKFLOW_ID")), }, "int": { "time": int(time.time()),