    001/*
    002 * Copyright (c) 2009 The openGion Project.
    003 *
    004 * Licensed under the Apache License, Version 2.0 (the "License");
    005 * you may not use this file except in compliance with the License.
    006 * You may obtain a copy of the License at
    007 *
    008 *     http://www.apache.org/licenses/LICENSE-2.0
    009 *
    010 * Unless required by applicable law or agreed to in writing, software
    011 * distributed under the License is distributed on an "AS IS" BASIS,
    012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
    013 * either express or implied. See the License for the specific language
    014 * governing permissions and limitations under the License.
    015 */
    016package org.opengion.cloud;
    017
    018import java.io.ByteArrayInputStream;
    019import java.io.File;
    020import java.io.FileFilter;
    021import java.io.FileNotFoundException;
    022import java.io.IOException;
    023import java.io.InputStream;
    024import java.util.ArrayList;
    025import java.util.List;
    026
    027import org.opengion.fukurou.model.FileOperation;
    028import org.opengion.fukurou.system.Closer;                                                      // 8.0.0.0 (2021/09/30) util.Closer → system.Closer
    029import org.opengion.fukurou.util.StringUtil;
    030import org.opengion.hayabusa.common.HybsSystem;
    031import org.opengion.hayabusa.common.HybsSystemException;
    032
    033import static org.opengion.fukurou.system.HybsConst.BUFFER_MIDDLE;      // HybsSystem.BUFFER_MIDDLE は fukurou に移動
    034import static org.opengion.fukurou.system.HybsConst.CR ;                        // 8.0.0.1 (2021/10/08)
    035
    036import com.amazonaws.auth.InstanceProfileCredentialsProvider;
    037import com.amazonaws.services.s3.AmazonS3;
    038import com.amazonaws.services.s3.AmazonS3ClientBuilder;
    039import com.amazonaws.services.s3.model.AmazonS3Exception;
    040import com.amazonaws.services.s3.model.ListObjectsV2Request;
    041import com.amazonaws.services.s3.model.ListObjectsV2Result;
    042import com.amazonaws.services.s3.model.ObjectListing;
    043import com.amazonaws.services.s3.model.ObjectMetadata;
    044import com.amazonaws.services.s3.model.PutObjectRequest;
    045import com.amazonaws.services.s3.model.S3Object;
    046import com.amazonaws.services.s3.model.S3ObjectSummary;
    047
    048/**
    049 * FileOperation_AWSは､S3ｽﾄﾚｰｼﾞに対して､
    050 * ﾌｧｲﾙ操作を行うｸﾗｽです｡
    051 *
    052 * 認証は下記の２通りが可能です｡→ １） の方法のみｻﾎﾟｰﾄします｡
    053 *  １）実行ｻｰﾊﾞのEC2のｲﾝｽﾀﾝｽに､S3ｽﾄﾚｰｼﾞのｱｸｾｽ許可を付与する
    054 * <del> ２）ｼｽﾃﾑﾘｿｰｽにｱｸｾｽｷｰ･ｼｰｸﾚｯﾄｷｰ･ｴﾝﾄﾞﾎﾟｲﾝﾄ･ﾚｷﾞｵﾝを登録する
    055 * （CLOUD_STORAGE_S3_ACCESS_KEY､CLOUD_STORAGE_S3_SECRET_KEY､CLOUD_STORAGE_S3_SERVICE_END_POINT､CLOUD_STORAGE_S3_REGION）
    056 * </del>
    057 *
    058 * 注意：
    059 * ﾊﾞｹｯﾄ名は全ﾕｰｻﾞで共有のため､自身のﾊﾞｹｯﾄ名か､作成されていないﾊﾞｹｯﾄ名を指定する必要があります｡
    060 *
    061 * @og.rev 5.10.8.0 (2019/02/01) 新規作成
    062 *
    063 * @version 5
    064 * @author  oota
    065 * @since   JDK7.0
    066 */
    067public class FileOperation_AWS extends CloudFileOperation {
    068        private static final long serialVersionUID = 5108020190201L ;
    069
    070        private static final String PLUGIN = "AWS";                     // 8.0.0.1 (2021/10/08) staticと大文字化
    071
    072        /** ｸﾗｽ変数 */
    073        private final AmazonS3 amazonS3;
    074
    075        /**
    076         * ｺﾝｽﾄﾗｸﾀｰ
    077         *
    078         * 初期化処理です｡
    079         * AWSの認証処理を行います｡
    080         *
    081         * @og.rev 8.0.0.1 (2021/10/08) CLOUD_STORAGE_S3… 関連の方法廃止
    082         *
    083         * @param bucket ﾊﾞｹｯﾄ
    084         * @param inPath ﾊﾟｽ
    085         */
    086        public FileOperation_AWS(final String bucket, final String inPath) {
    087                super(StringUtil.nval(bucket, HybsSystem.sys("CLOUD_BUCKET")), inPath);
    088
    089                // IAMﾛｰﾙによる認証
    090                amazonS3 = AmazonS3ClientBuilder.standard()
    091                                .withCredentials(new InstanceProfileCredentialsProvider(false))
    092                                .build();
    093
    094                try {
    095                        // S3に指定されたﾊﾞｹｯﾄ(ｺﾝﾃﾅ)が存在しない場合は､作成する
    096                        if (!amazonS3.doesBucketExistV2(conBucket)) { // doesBucketExistV2最新JARだと出ている
    097                                amazonS3.createBucket(conBucket);
    098                        }
    099                } catch (final AmazonS3Exception ase) {
    100                        final String errMsg = new StringBuilder(BUFFER_MIDDLE)
    101                                                        .append("IAMﾛｰﾙによる認証が失敗しました｡").append( CR )
    102                                                        .append( inPath ).toString();
    103
    104                        throw new HybsSystemException(errMsg,ase);
    105                }
    106        }
    107
    108        /**
    109         * 書き込み処理(評価用)
    110         *
    111         * Fileを書き込みます｡
    112         *
    113         * @og.rev 8.0.0.1 (2021/10/08) 新規追加
    114         *
    115         * @param inFile 書き込みFile
    116         * @throws IOException ﾌｧｲﾙ関連ｴﾗｰ情報
    117         */
    118        @Override
    119        public void write(final File inFile) throws IOException {
    120                try {
    121                        amazonS3.putObject(conBucket, conPath, inFile);
    122                } catch (final Exception ex) {
    123                        final String errMsg = new StringBuilder(BUFFER_MIDDLE)
    124                                                .append("AWSﾊﾞｹｯﾄに(File)書き込みが失敗しました｡").append( CR )
    125                                                .append(conPath).toString();
    126                        throw new IOException(errMsg,ex);
    127                }
    128        }
    129
    130        /**
    131         * 書き込み処理
    132         *
    133         * InputStreamのﾃﾞｰﾀを書き込みます｡
    134         *
    135         * @param is 書き込みﾃﾞｰﾀのInputStream
    136         * @throws IOException ﾌｧｲﾙ関連ｴﾗｰ情報
    137         */
    138        @Override
    139        public void write(final InputStream is) throws IOException {
    140                ByteArrayInputStream bais = null;
    141                try {
    142                        final ObjectMetadata om = new ObjectMetadata();
    143
    144                        final byte[] bytes = toByteArray(is);
    145                        om.setContentLength(bytes.length);
    146                        bais = new ByteArrayInputStream(bytes);
    147
    148                        final PutObjectRequest request = new PutObjectRequest(conBucket, conPath, bais, om);
    149
    150                        amazonS3.putObject(request);
    151                } catch (final Exception ex) {
    152                        final String errMsg = new StringBuilder(BUFFER_MIDDLE)
    153                                                .append("AWSﾊﾞｹｯﾄに(InputStream)書き込みが失敗しました｡").append( CR )
    154                                                .append(conPath).toString();
    155                        throw new IOException(errMsg,ex);
    156                } finally {
    157                        Closer.ioClose(bais);
    158                }
    159        }
    160
    161        /**
    162         * 読み込み処理
    163         *
    164         * ﾃﾞｰﾀを読み込み､InputStreamとして､返します｡
    165         *
    166         * @return 読み込みﾃﾞｰﾀのInputStream
    167         * @throws FileNotFoundException ﾌｧｲﾙ非存在ｴﾗｰ情報
    168         */
    169        @Override
    170        public InputStream read() throws FileNotFoundException {
    171                S3Object object = null;
    172
    173                try {
    174                        object = amazonS3.getObject(conBucket, conPath);
    175                } catch (final Exception ex) {
    176                        final String errMsg = new StringBuilder(BUFFER_MIDDLE)
    177                                                .append("AWSﾊﾞｹｯﾄから読み込みが失敗しました｡").append( CR )
    178                                                .append(conPath).append( CR )
    179                                                .append( ex.getMessage() ).toString();
    180                        throw new FileNotFoundException(errMsg);
    181                }
    182                return object.getObjectContent();       // com.amazonaws.services.s3.model.S3ObjectInputStream
    183        }
    184
    185        /**
    186         * 削除処理
    187         *
    188         * ﾌｧｲﾙを削除します｡
    189         *
    190         * @return 成否ﾌﾗｸﾞ
    191         */
    192        @Override
    193        public boolean delete() {
    194                boolean flgRtn = false;
    195
    196                try {
    197                        if (isFile()) {
    198                                // ﾌｧｲﾙ削除
    199                                amazonS3.deleteObject(conBucket, conPath);
    200                        } else if (isDirectory()) {
    201                                // ﾃﾞｨﾚｸﾄﾘ削除
    202                                // 一括削除のapiが無いので､繰り返しで削除を行う
    203                                final ObjectListing objectList = amazonS3.listObjects(conBucket, conPath);
    204                                final List<S3ObjectSummary> list = objectList.getObjectSummaries();
    205                                for (final S3ObjectSummary obj : list) {
    206                                        amazonS3.deleteObject(conBucket, obj.getKey());
    207                                }
    208                        }
    209                        flgRtn = true;
    210                } catch (final Exception ex) {
    211                        // ｴﾗｰはｽﾙｰして､falseを返す
    212                        System.out.println( ex.getMessage() );
    213                }
    214
    215                return flgRtn;
    216        }
    217
    218        /**
    219         * ｺﾋﾟｰ処理
    220         *
    221         * ﾌｧｲﾙを指定先に､ｺﾋﾟｰします｡
    222         *
    223         * @param afPath ｺﾋﾟｰ先
    224         * @return 成否ﾌﾗｸﾞ
    225         */
    226        @Override
    227        public boolean copy(final String afPath) {
    228                boolean flgRtn = false;
    229
    230                try {
    231                        amazonS3.copyObject(conBucket, conPath, conBucket, afPath);
    232                        flgRtn = true;
    233                } catch (final Exception ex) {
    234                        // ｴﾗｰはｽﾙｰして､falseを返す
    235                        System.out.println( ex.getMessage() );
    236                }
    237
    238                return flgRtn;
    239        }
    240
    241        /**
    242         * ﾌｧｲﾙｻｲｽﾞ取得
    243         *
    244         * ﾌｧｲﾙｻｲｽﾞを返します｡
    245         *
    246         * @return ﾌｧｲﾙｻｲｽﾞ
    247         */
    248        @Override
    249        public long length() {
    250                long rtn = 0;
    251
    252                try {
    253                        final ObjectMetadata meta = amazonS3.getObjectMetadata(conBucket, conPath);
    254                        rtn = meta.getContentLength();
    255                } catch (final Exception ex) {
    256                        // ｴﾗｰはｽﾙｰして､0を返す｡
    257                        System.out.println( ex.getMessage() );
    258                }
    259                return rtn;
    260        }
    261
    262        /**
    263         * 最終更新時刻取得
    264         *
    265         * 最終更新時刻を取得します｡
    266         *
    267         * @return 最終更新時刻
    268         */
    269        @Override
    270        public long lastModified() {
    271                long rtn = 0;
    272
    273                try {
    274                        final ObjectMetadata meta = amazonS3.getObjectMetadata(conBucket, conPath);
    275                        rtn = meta.getLastModified().getTime();
    276                } catch (final Exception ex) {
    277                        // ｴﾗｰはｽﾙｰして､0を返す
    278                        System.out.println( ex.getMessage() );
    279                }
    280                return rtn;
    281        }
    282
    283        /**
    284         * ﾌｧｲﾙ判定
    285         *
    286         * ﾌｧｲﾙの場合は､trueを返します｡
    287         *
    288         * @return ﾌｧｲﾙ判定ﾌﾗｸﾞ
    289         */
    290        @Override
    291        public boolean isFile() {
    292                boolean rtn = false;
    293
    294                if(!isDirectory()) {
    295                        rtn = amazonS3.doesObjectExist(conBucket, conPath);
    296                }
    297
    298                return rtn;
    299        }
    300
    301        /**
    302         * ﾃﾞｨﾚｸﾄﾘ判定
    303         *
    304         * ﾃﾞｨﾚｸﾄﾘの場合は､trueを返します｡
    305         *
    306         * @return ﾃﾞｨﾚｸﾄﾘ判定ﾌﾗｸﾞ
    307         */
    308        @Override
    309        public boolean isDirectory() {
    310                if (StringUtil.isEmpty(conPath)) {              // 8.0.1.0 (2021/10/29) org.apache.commons.lang3.StringUtils 置換
    311                        return true;
    312                }
    313
    314                // S3にはﾃﾞｨﾚｸﾄﾘの概念はないので､｢/｣で続くﾃﾞｰﾀが存在するかで､判定
    315                final ObjectListing objectList = amazonS3.listObjects(conBucket, setDirTail(conPath));
    316                final List<S3ObjectSummary> list = objectList.getObjectSummaries();
    317
    318                return list.size() != 0 ;
    319        }
    320
    321        /**
    322         * ﾌｧｲﾙ一覧取得
    323         *
    324         * ﾊﾟｽのﾌｧｲﾙとﾃﾞｨﾚｸﾄﾘ一覧を取得します｡
    325         *
    326         * @param filter ﾌｨﾙﾀ情報
    327         * @return ﾌｧｲﾙとﾃｨﾚｸﾄﾘ一覧
    328         */
    329        @Override
    330        public File[] listFiles(final FileFilter filter) {
    331                if (!exists()) {
    332                        return new FileOperationInfo[0];
    333                }
    334
    335                String search = conPath;
    336                if (isDirectory()) {
    337                        search = setDirTail(conPath);
    338                }
    339
    340                final List<File> rtnList = new ArrayList<File>();
    341
    342                // 検索処理
    343                final ListObjectsV2Request request = new ListObjectsV2Request()
    344                                .withBucketName(conBucket)
    345                                .withPrefix(search)
    346                                .withDelimiter("/");
    347                final ListObjectsV2Result list = amazonS3.listObjectsV2(request);
    348                final List<S3ObjectSummary> objects = list.getObjectSummaries();
    349
    350                // ﾌｧｲﾙ情報の取得
    351                for (final S3ObjectSummary obj : objects) {
    352                        final String key = obj.getKey();
    353
    354                        final FileOperationInfo file = new FileOperationInfo(PLUGIN, conBucket, key);
    355                        file.setLastModifiedValue(obj.getLastModified().getTime());
    356                        file.setFile(true);
    357                        file.setSize(obj.getSize());
    358                        rtnList.add(file);
    359                }
    360
    361                // ﾃﾞｨﾚｸﾄﾘ情報の取得
    362                final List<String> folders = list.getCommonPrefixes();
    363                for (final String str : folders) {
    364                        final String key = rTrim(str, '/');
    365
    366                        final FileOperationInfo file = new FileOperationInfo(PLUGIN, conBucket, key);
    367                        file.setDirectory(true);
    368                        rtnList.add(file);
    369                }
    370
    371                // ﾌｨﾙﾀ処理
    372                return filter(rtnList, filter);
    373        }
    374
    375        /**
    376         * 親ﾃﾞｨﾚｸﾄﾘ情報の取得
    377         *
    378         * 親のﾃﾞｨﾚｸﾄﾘを返します｡
    379         *
    380         * @return 親のﾃﾞｨﾚｸﾄﾘ情報
    381         */
    382        @Override
    383        public FileOperation getParentFile() {
    384                return new FileOperation_AWS(conBucket, this.getParent());
    385        }
    386}