aliyun☞oss

阅读量: zyh 2020-11-24 16:48:43

Categories： > Tags：

构建 oss 存储桶

执行脚本（执行本脚本跑到阿里云的云端cli去执行）
执行前应先复制修改下面的策略模板

#!/usr/bin/python3.6
# -*- coding: utf-8 -*-
#
##  Usage：https://help.aliyun.com/document_detail/32027.html
##  Github：https://github.com/aliyun/aliyun-oss-python-sdk
##  author: zyh

import oss2, os
from oss2.models import (LifecycleExpiration, LifecycleRule,
                        BucketLifecycle,AbortMultipartUpload,
                        TaggingRule, Tagging, StorageTransition,
                        NoncurrentVersionStorageTransition,
                        NoncurrentVersionExpiration)
from oss2.models import Tagging, TaggingRule
#
from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.acs_exception.exceptions import ClientException
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkram.request.v20150501.CreatePolicyRequest import CreatePolicyRequest
#
#
##################################
region = '我是区域ID'
bucketName = '我是桶名'
project = '我是标签project的值'
vpcNetwork = '我是桶所在大区的ECS的内网网段'
akey = 
skey = 
##################################
#
endpoint = 'http://oss-{0}.aliyuncs.com'.format(region)
auth = oss2.Auth(akey,skey)
bucket = oss2.Bucket(auth, endpoint, bucketName)

# create bucket
bucket.create_bucket()

# add tag
rule = TaggingRule()
rule.add('project', project)
tagging = Tagging(rule)
bucket.put_bucket_tagging(tagging)

# init dirs
bucket.put_object('conf/README','我是存放配置的目录')
bucket.put_object('data/README','我是存放数据的目录')
bucket.put_object('hive/README','我是存放hive的目录')
bucket.put_object('backup/README','我是存放备份的目录')
bucket.put_object('logs/7days/README','我是存放保留7天的日志目录')
bucket.put_object('logs/15days/README','我是存放保留15天的日志目录')
bucket.put_object('logs/30days/README','我是存放保留30天的日志目录')
bucket.put_object('logs/60days/README','我是存放保留60天的日志目录')
bucket.put_object('logs/90days/README','我是存放保留90天的日志目录')
bucket.put_object('logs/180days/README','我是存放永久保留的日志目录')

# add lifecycle
rule0 = LifecycleRule('rule0', 'tmp/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=3))
rule1 = LifecycleRule('rule1', 'logs/7days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=7))
rule2 = LifecycleRule('rule2', 'logs/15days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=15))
rule3 = LifecycleRule('rule3', 'logs/30days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=30))
rule4 = LifecycleRule('rule4', 'logs/60days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=60))
rule5 = LifecycleRule('rule5', 'logs/90days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=90))
rule6 = LifecycleRule('rule6', 'logs/180days/',
                      status=LifecycleRule.ENABLED,
                      expiration=LifecycleExpiration(days=180))
rule7 = LifecycleRule('rule7', 'logs/longlasting/',
                      status=LifecycleRule.ENABLED,
                      storage_transitions=[StorageTransition(days=60,storage_class=oss2.BUCKET_STORAGE_CLASS_IA),
                          StorageTransition(days=180,storage_class=oss2.BUCKET_STORAGE_CLASS_ARCHIVE)])

lifecycle = BucketLifecycle([rule0, rule1, rule2, rule3, rule4, rule5, rule6, rule7])
bucket.put_bucket_lifecycle(lifecycle)


os.system("sed 's#ossBucketName#{0}#g' local.policy.default > local/local_{0}.policy".format(bucketName))
os.system("sed 's#ossBucketName#{0}#g' role.policy.default > local/role_{0}.policy".format(bucketName))
os.system("sed 's#ossBucketName#{0}#g' apps.policy.default > local/apps_{0}.policy".format(bucketName))
os.system("sed 's#0.0.0.0#{0}#g' apps.policy.default > local/apps_{1}.policy".format(vpcNetwork, bucketName))
os.system("aliyun ram CreatePolicy --PolicyName oss-{0}-local --PolicyDocument \"`cat local/local_{0}.policy`\"".format(bucketName))
os.system("aliyun ram CreatePolicy --PolicyName oss-{0}-role --PolicyDocument \"`cat local/role_{0}.policy`\"".format(bucketName))
os.system("aliyun ram CreatePolicy --PolicyName oss-{0}-apps --PolicyDocument \"`cat local/apps_{0}.policy`\"".format(bucketName))

远程用户策略（执行脚本前需要添加的）

local.policy.default

✨自行替换策略里的1.1.1.1为公司ip或者远程ip

{
  "Version": "1",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
                "oss:GetObject",
                "oss:PutObject",
                "oss:DeleteObject",
                "oss:ListObjects"
      ],
      "Resource": [
        "acs:oss:*:*:ossBucketName/logs/7days/*",
        "acs:oss:*:*:ossBucketName/logs/15days/*",
        "acs:oss:*:*:ossBucketName/logs/30days/*",
        "acs:oss:*:*:ossBucketName/logs/60days/*",
        "acs:oss:*:*:ossBucketName/logs/90days/*",
        "acs:oss:*:*:ossBucketName/logs/180days/*",
        "acs:oss:*:*:ossBucketName/logs/longlasting/*",
        "acs:oss:*:*:ossBucketName/conf/*",
        "acs:oss:*:*:ossBucketName/data/*",
        "acs:oss:*:*:ossBucketName/backup/*",
        "acs:oss:*:*:ossBucketName/hive/*",
        "acs:oss:*:*:ossBucketName/tmp/*",
        "acs:oss:*:*:ossBucketName"
      ],
      "Condition": {
        "IpAddress": {
          "acs:SourceIp": [
            "1.1.1.1"
          ]
        }
      }
    },
    {
      "Effect": "Allow",
      "Action": [
        "oss:ListBuckets",
        "oss:GetBucketStat",
        "oss:GetBucketInfo",
        "oss:GetBucketTagging",
        "oss:GetBucketAcl",
        "oss:GetBucketLocation"
      ],
      "Resource": [
        "acs:oss:*:*:*"
      ],
      "Condition": {
        "IpAddress": {
          "acs:SourceIp": [
            "1.1.1.1"
          ]
        }
      }
    }
  ]
}

角色用户策略（执行脚本前需要添加的）

角色策略不限制源，用于绑定到阿里云资源上，例如ECS

程序通过调用角色来获取临时权限

{
  "Version": "1",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
                "oss:GetObject",
                "oss:PutObject",
                "oss:DeleteObject",
                "oss:ListObjects"
      ],
      "Resource": [
        "acs:oss:*:*:ossBucketName/logs/7days/*",
        "acs:oss:*:*:ossBucketName/logs/15days/*",
        "acs:oss:*:*:ossBucketName/logs/30days/*",
        "acs:oss:*:*:ossBucketName/logs/60days/*",
        "acs:oss:*:*:ossBucketName/logs/90days/*",
        "acs:oss:*:*:ossBucketName/logs/180days/*",
        "acs:oss:*:*:ossBucketName/logs/longlasting/*",
        "acs:oss:*:*:ossBucketName/conf/*",
        "acs:oss:*:*:ossBucketName/data/*",
        "acs:oss:*:*:ossBucketName/backup/*",
        "acs:oss:*:*:ossBucketName/hive/*",
        "acs:oss:*:*:ossBucketName/tmp/*"
        "acs:oss:*:*:ossBucketName"
      ]
    },
    {
      "Effect": "Allow",
      "Action": [
        "oss:ListBuckets",
        "oss:GetBucketStat",
        "oss:GetBucketInfo",
        "oss:GetBucketTagging",
        "oss:GetBucketAcl",
        "oss:GetBucketLocation"
      ],
      "Resource": [
        "acs:oss:*:*:*"
      ]
    }
  ]
}

程序用户策略

用于ecs里的程序调用AK方式

{
  "Version": "1",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
                "oss:GetObject",
                "oss:PutObject",
                "oss:DeleteObject",
                "oss:ListObjects"
      ],
      "Resource": [
        "acs:oss:*:*:ossBucketName/logs/7days/*",
        "acs:oss:*:*:ossBucketName/logs/15days/*",
        "acs:oss:*:*:ossBucketName/logs/30days/*",
        "acs:oss:*:*:ossBucketName/logs/60days/*",
        "acs:oss:*:*:ossBucketName/logs/90days/*",
        "acs:oss:*:*:ossBucketName/logs/180days/*",
        "acs:oss:*:*:ossBucketName/logs/longlasting/*",
        "acs:oss:*:*:ossBucketName/conf/*",
        "acs:oss:*:*:ossBucketName/data/*",
        "acs:oss:*:*:ossBucketName/backup/*",
        "acs:oss:*:*:ossBucketName/hive/*",
        "acs:oss:*:*:ossBucketName/tmp/*",
        "acs:oss:*:*:ossBucketName"
      ],
      "Condition": {
        "IpAddress": {
          "acs:SourceIp": [
            "0.0.0.0"
          ]
        }
      }
    },
    {
      "Effect": "Allow",
      "Action": [
        "oss:ListBuckets",
        "oss:GetBucketStat",
        "oss:GetBucketInfo",
        "oss:GetBucketTagging",
        "oss:GetBucketAcl",
        "oss:GetBucketLocation"
      ],
      "Resource": [
        "acs:oss:*:*:*"
      ],
      "Condition": {
        "IpAddress": {
          "acs:SourceIp": [
            "0.0.0.0"
          ]
        }
      }
    }
  ]
}

使用

以角色授权方式+aliyun cli命令方式走起.

😔，aliyun cli 的文档和使用一言难尽=。=

使用前请确保角色已经关联了对应权限以及角色已经绑定到了ECS上

# /etc/profile.d/ecs_role.sh
Region=`curl -sq http://100.100.100.200/latest/meta-data/region-id`
ramRoleName=`curl -sq http://100.100.100.200/latest/meta-data/ram/security-credentials/`
aliyun configure set --profile ecsRamRoleProfile  --mode EcsRamRole --ram-role-name ${ramRoleName} --region ${Region}

本脚本，会让任何一个会话登陆的时候就拿到角色拥有的权限

导入角色

##导入角色，获取权限以及一些变量
source /etc/profile.d/ecs_role.sh

设置端点

## 设定 oss 的 endpoint 地址, EndpointLan VPC内使用，EndpointWan VPC外使用
EndpointLan="http://oss-${Region}-internal.aliyuncs.com"
EndpointWan="http://oss-${Region}.aliyuncs.com"

进行操作

##查询
## 默认查询是递归查询，-d 只查询一层
aliyun oss ls oss://test/ -d -e ${EndpointLan}

##基本的上传或下载
##上传文件 a.file 到 oss://test/ 
aliyun oss cp a.file oss://test/ -e ${EndpointLan}

##基本的递归上传
##上传目录 abc 下的文件到 oss://test/ 下，如果有重复内容，则需要加入 --force
aliyun oss cp abc oss://test/ --recursive -e ${EndpointLan}

##复杂的递归上传
##上传目录 abc 下的 .lzo 结尾的文件到 oss://test/ 下.
##严禁在源目录里执行 --recursive 参数.
##即禁止执行 aliyun oss cp . oss://test/ --recursive 
aliyun oss cp abc/ oss://test/ --include='*.lzo' --update --recursive -e ${EndpointLan} --checkpoint-dir=/tmp/ossutil_checkpoint --output-dir=/tmp/ossutil_output

##同步目录 sync 指令变更
##同步目录 abc 下的文件到 oss://test/ 下，如有重复，则忽略;同时删除目标目录下本地没有的文件
aliyun oss sync abc/ oss://test/ --update --delete --force -e ${EndpointLan} --checkpoint-dir=/tmp/ossutil_checkpoint --output-dir=/tmp/ossutil_output

开发向 sdk

关于php sdk访问对象存储的文档

php oss 对象 sdk
https://packagist.org/packages/aliyuncs/oss-sdk-php?spm=a2c6h.13321295.0.0.4f765c2dMbvzR6
php ram role sdk
https://packagist.org/packages/alibabacloud/credentials?spm=a2c6h.13321295.0.0.4f765c2dMbvzR6

go-016并发

go-015自定义包