AWSでs3バケット利用

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

{
  "Version": "2012-10-17", ⭐️日付は気にしない、こういうもんや
  "Statement": [
    {
      "Sid": "VisualEditor0",
      "Effect": "Allow", ⭐️表示・読み書き・削除を許可する
      "Action": [
        "s3:ListBucket",
        "s3:GetObject*",
        "s3:PutObject*",
        "s3:DeleteObject*"
      ],
      "Resource": [  ⭐️対象バケットからarnで指定
        "arn:aws:s3:::*nari-s3-bucket*",
        "arn:aws:s3:::*nari-s3-bucket*/*"
      ]
    }
  ]
}

1
2
3
4
5
6
7
8
9

; script base folder
SCHOME=D:\_GVIS\
; aws s3 name
AWS_S3_BUCKET=nari-s3-bucket
; aws s3 bandwidth
; 帯域食いつぶさないように設定。1000MB/sだと10GBのデータでも1分ぐらいで終了
AWS_S3_BANDWIDTH=1000MB/s
; ログ保管期間
LOG_KEEP=7

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

@echo off
rem ## -------------------------------------------------------------------------
rem ##        Name  : s3backup.bat
rem ##  Created by  : gvis
rem ##          on  : 2025.7.28
rem ##  Updated by  :
rem ##          on  :
rem ##     Comments : S3バケットへアップロード
rem ##         Args : 1)バックアップ対象フォルダ指定(例: D:\backup)
rem ##              : 2)保管先s3フォルダ指定(例: backup)
rem ##         ret  : 0:normal end other:abend
rem ##
rem ## -------------------------------------------------------------------------

rem ## ---set env---------------------------------------------------------------

for /f "tokens=1,* delims==" %%a in (D:\gvis\variable.ini) do (
    set %%a=%%b
)

set DATESTR=%date:~0,4%%date:~5,2%%date:~8,2%

set LOGFOLDER=%SCHOME%log

rem ログフォルダがなければ作成
if not exist "%SCHOME%log" (
mkdir %LOGFOLDER%
)

set LOGFILE=%LOGFOLDER%\LOG-%DATESTR%.log

echo DATESTR=%DATESTR% >> %LOGFILE%
echo LOGFILE=%LOGFILE% >> %LOGFILE%
echo SCHOME=%SCHOME% >> %LOGFILE%
echo AWS_S3_BUCKET=%AWS_S3_BUCKET% >> %LOGFILE%
echo AWS_S3_BANDWIDTH=%AWS_S3_BANDWIDTH% >> %LOGFILE%

rem コンピュータ名取得
set strHost=%COMPUTERNAME%

set s3path=s3://%AWS_S3_BUCKET%/dailyExport/%COMPUTERNAME%/%DATESTR%

echo s3path=%s3path% >> %LOGFILE%

set BACKUP_FOLDER=%1

echo BACKUP_FOLDER=%1 >> %LOGFILE%

set TARGET_FOLDER=%2

echo TARGET_FOLDER=%2 >> %LOGFILE%

echo. >> %LOGFILE%

rem 引数のチェック
if "%1"=="" (
  echo 引数1がありません >> %LOGFILE%
  exit 1
)

if "%2"=="" (
  echo 引数2がありません >> %LOGFILE%
  exit 1
)

rem 引数のフォルダチェック

if not exist %BACKUP_FOLDER% (
  echo 引数指定のフォルダがありません　 >> %LOGFILE%
  exit 2
)

rem 処理開始 ====================================================
echo ------------------start %time: =% --------------------------------  >> %LOGFILE%
echo ■バックアップ開始　 >> %LOGFILE%

echo □データコピー「%BACKUP_FOLDER%」⇒「%s3path%」  >> %LOGFILE%

rem --同期
rem aws s3 sync %BACKUP_FOLDER% %s3path% --delete  >> %LOGFILE% 2>&1

rem --フォルダ再帰削除
aws configure set default.s3.max_bandwidth %AWS_S3_BANDWIDTH%
aws s3 rm %s3path%/%TARGET_FOLDER%                 --recursive >> %LOGFILE% 2>&1
echo -----------------cleaned %time: =% -------------------------------  >> %LOGFILE%

rem --フォルダ再帰コピー ※ntfsやxfsのファイルシステムと違って空フォルダは維持されない
aws s3 cp %BACKUP_FOLDER% %s3path%/%TARGET_FOLDER% --recursive >> %LOGFILE% 2>&1
if %errorlevel% geq 1 goto ERROREND

echo ------------------finish %time: =% -------------------------------  >> %LOGFILE%
exit /b 0

:ERROREND
echo ------------------finish %time: =% -------------------------------  >> %LOGFILE%
echo ログを確認してください  >> %LOGFILE%
exit /b 99

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

@echo off
rem ## -------------------------------------------------------------------------
rem ##        Name  : rotate.bat
rem ##  Created by  : gvis
rem ##          on  : 2023.8.5
rem ##  Updated by  :
rem ##          on  :
rem ##  Parameters  :
rem ##     Comments : ログ洗い替え
rem ##         Args : なし
rem ##
rem ## -------------------------------------------------------------------------

rem ## ---set env---------------------------------------------------------------

for /f "tokens=1,* delims==" %%a in (D:\gvis\variable.ini) do (
    set %%p=%%q
)

echo SCHOME=%SCHOME%
echo LOG_KEEP=%LOG_KEEP%

rem ## ---detail----------------------------------------------------------------

forfiles /p %SCHOME%log /d -%LOG_KEEP% /m "*.log" /c "cmd /c del @file"

rem pause
exit 0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

{
    "Version": "2012-10-17",
    "Id": "AWSLogDeliveryWrite",
    "Statement": [
        {
            "Sid": "AWSLogDeliveryAclCheck",
            "Effect": "Allow",
            "Principal": {
                "Service": "delivery.logs.amazonaws.com"
            },
            "Action": "s3:GetBucketAcl",
            "Resource": "arn:aws:s3:::nari-s3-bucket",
            "Condition": {
                "StringEquals": {
                    "aws:SourceAccount": "xxxxxxxxxxxx" ⭐️awsアカウントの１２桁を指定
                },
                "ArnLike": {
                    "aws:SourceArn": "arn:aws:logs:ap-northeast-1:xxxxxxxxxxxx:*" ⭐️リージョンとawsアカウントの１２桁を指定
                }
            }
        },
        {
            "Sid": "AWSLogDeliveryWrite",
            "Effect": "Allow",
            "Principal": {
                "Service": "delivery.logs.amazonaws.com"
            },
            "Action": "s3:PutObject",
            "Resource": "arn:aws:s3:::nari-s3-bucket/*", ⭐️バケット名を指定
            "Condition": {
                "StringEquals": {
                    "s3:x-amz-acl": "bucket-owner-full-control",
                    "aws:SourceAccount": "xxxxxxxxxxxx" ⭐️awsアカウントの１２桁を指定
                },
                "ArnLike": {
                    "aws:SourceArn": "arn:aws:logs:ap-northeast-1:xxxxxxxxxxxx:*" ⭐️リージョンとawsアカウントの１２桁を指定
                }
            }
        }
    ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

; script base folder
SCHOME=D:\gvis\
; sqlplus fullpath
SQP=D:\gvis\oracleClient\sqlplus.exe ⭐️実行のためのフルパスを設定
; oracle connect
D_ORACON=nariRDS-DB.ue06nsqi2hal.ap-northeast-1.rds.amazonaws.com/nariRDS-DB ⭐️rds-oracleのデータベース
; oracle auth user
D_ORAUSER=admin
; oracle auth pass
D_ORAPASS=xxx

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

rem ^ ⭐️この記述必要 - 次の行をコメント行として判断させるためにこう書く
/* ⭐️この記述必要 - sqlplusに渡したとき、ここからコメントと認識させるため
@echo off
rem ## -------------------------------------------------------------------------
rem ##        Name  : oraExp.bat
rem ##  Created by  : gvis
rem ##          on  : 2024.6.28
rem ##  Updated by  :
rem ##          on  :
rem ##  Parameters  :
rem ##     Comments : oracleエクスポート起動
rem ##         Args : スキーマ名、日付(yyyymmdd)
rem ##
rem ## -------------------------------------------------------------------------

rem ## ---set env---------------------------------------------------------------

for /f "tokens=1,* delims==" %%a in (E:\gvis\variable.ini) do (
    set %%p=%%q
)

rem set DATESTR=%date:~0,4%%date:~5,2%%date:~8,2%
set DATESTR=%2

set LOGFILE=%SCHOME%log\LOG-ORAEXP-%DATESTR%.log
set RC=0

echo SCHOME=%SCHOME%
echo SQP=%SQP%
echo DATESTR=%DATESTR%
echo LOGFILE=%LOGFILE%
echo D_ORACON=%D_ORACON%
echo D_ORAUSER=%D_ORAUSER%
echo D_ORAPASS=%D_ORAPASS%

rem ## ---detail----------------------------------------------------------------

echo ------------------start %time: =% -------------------------------- >> %LOGFILE%

rem ##################################################################################
rem #                                                                                #
rem # 接続先と定数を指定する                                                     　　#
rem #                                                                                #
rem ##################################################################################
set CONN=%D_ORAUSER%/%D_ORAPASS%@%D_ORACON%

rem ##################################################################################
rem #                                                                                #
rem # バックアップ対象のスキーマは引数で指定する                                     #
rem # テスト時は直接指定する                                                     　　#
rem #                                                                                #
rem ##################################################################################
set BACKUPSCHEMA=%1

rem ##################################################################################
rem #                                                                                #
rem # 外部ファイルではなくバッチファイルそのものをsqlファイルとして取り込んで実行    #
rem # 「／＊」「＊／」で囲った範囲と「rem」で始まる行はsqlplus取り込んでも無視される #
rem #                                                                                #
rem ##################################################################################

echo EXEC_CMD=%SQP% %CONN% @"%~f0" %BACKUPSCHEMA%
echo EXEC_CMD=%SQP% %CONN% @"%~f0" %BACKUPSCHEMA% >> %LOGFILE%
rem pause

%SQP% %CONN% @"%~f0" %BACKUPSCHEMA% >> %LOGFILE% ⭐️sqlplusに接続情報、バッチファイル名本体のフルパス名、スキーマ名渡す

echo ------------------finish %time: =% ------------------------------- >> %LOGFILE%

rem pause

exit
*/ ⭐️この記述必要 - sqlplusに渡してここまでをコメントとして認識させるためで、exitの後ろに書いとけばエラー認識されん

set pagesize 10000
set lin 100
set wrap off

ALTER SESSION SET NLS_DATE_FORMAT = 'YYYYMMDD' ;
spool E:\gvis\log\spool-&_DATE..log

DECLARE
  v_hdnl NUMBER;
  status VARCHAR2(20);
BEGIN

  v_hdnl := DBMS_DATAPUMP.OPEN(
    operation => 'EXPORT',
    job_mode  => 'SCHEMA',
    job_name  => null
  );
  DBMS_DATAPUMP.ADD_FILE(
    handle    => v_hdnl,
    filename  => '&1..DMP', ⭐️ダンプファイルにスキーマ名つける
    directory => 'DATA_PUMP_DIR',
    filetype  => dbms_datapump.ku$_file_type_dump_file,
    reusefile => 1
  );
  DBMS_DATAPUMP.ADD_FILE(
    handle    => v_hdnl,
    filename  => '&1._exp.log', ⭐️ダンプの実行ログにスキーマ名を混ぜる
    directory => 'DATA_PUMP_DIR' ,
    filetype  => dbms_datapump.ku$_file_type_log_file,
    reusefile => 1
  );
  DBMS_DATAPUMP.METADATA_FILTER(v_hdnl,'SCHEMA_EXPR','IN (''&1'')');
  DBMS_DATAPUMP.START_JOB(v_hdnl);
  DBMS_DATAPUMP.WAIT_FOR_JOB(v_hdnl,status);
END;
/

spool off

EXIT