Difference between revisions of "NextCloud: Heroku: Install: Apply S3 Patch to NextCloud"

From Gratiel's Wiki
(→‎Code Block 7)
(→‎Apply the Patch)
 
(8 intermediate revisions by the same user not shown)
Line 4: Line 4:
  
 
== Manually Applying Code ==
 
== Manually Applying Code ==
 +
# Bring up this file inside an editor.
 +
# In each Block Section of Code:
 +
## Navigate to the specified line number
 +
## <span style='background-color: #fbe9eb; border: #f44256 4px solid;'>Remove any line of code highlighted in RED</span>
 +
## <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>Add any new line of code highlighted in GREEN<span>
  
 
=== Code Block 1 ===
 
=== Code Block 1 ===
Line 38: Line 43:
 
         }
 
         }
 
     }
 
     }
−
     unset($this->filesCache[$key]);
+
     <span style='background-color: #fbe9eb; border: #f44256 4px solid;'>unset($this->filesCache[$key]);</span>
 
     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>unset($this->directoryCache[$key], $this->filesCache[$key]);</span>
 
     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>unset($this->directoryCache[$key], $this->filesCache[$key]);</span>
 
  }
 
  }
Line 47: Line 52:
 
     }
 
     }
 
   
 
   
−
     /**
+
     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>/**</span>
−
       * Return true if directory exists
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* Return true if directory exists</span>
−
       *
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>*</span>
−
       * There are no folders in s3. A folder like structure could be archived
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* There are no folders in s3. A folder like structure could be archived</span>
−
       * by prefixing files with the folder name.
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* by prefixing files with the folder name.</span>
−
       *
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>*</span>
−
       * Implementation from flysystem-aws-s3-v3:
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* Implementation from flysystem-aws-s3-v3:</span>
−
       * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694</span>
−
       *
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>*</span>
−
       * @param $path
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* @param $path</span>
−
       * @return bool
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* @return bool</span>
−
       * @throws \Exception
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>* @throws \Exception</span>
−
       */
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>*/</span>
−
     private function doesDirectoryExist($path) {
+
     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>private function doesDirectoryExist($path) {</span>
−
         if (!isset($this->directoryCache[$path])) {
+
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>if (!isset($this->directoryCache[$path])) {</span>
−
             // Maybe this isn't an actual key, but a prefix.
+
             <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>// Maybe this isn't an actual key, but a prefix.</span>
−
             // Do a prefix listing of objects to determine.
+
             <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>// Do a prefix listing of objects to determine.</span>
−
             try {
+
             <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>try {</span>
−
                 $result = $this->getConnection()->listObjects([
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$result = $this->getConnection()->listObjects([</span>
−
                     'Bucket' => $this->bucket,
+
                     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>'Bucket' => $this->bucket,</span>
−
                     'Prefix' => rtrim($path, '/') . '/',
+
                     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>'Prefix' => rtrim($path, '/') . '/',</span>
−
                     'MaxKeys' => 1,
+
                     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>'MaxKeys' => 1,</span>
−
                 ]);
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>]);</span>
−
                 $this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];</span>
−
             } catch (S3Exception $e) {
+
             <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>} catch (S3Exception $e) {</span>
−
                 if ($e->getStatusCode() === 403) {
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>if ($e->getStatusCode() === 403) {</span>
−
                     $this->directoryCache[$path] = false;
+
                     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$this->directoryCache[$path] = false;</span>
−
                 }
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>}</span>
−
                 throw $e;
+
                 <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>throw $e;</span>
−
             }
+
             <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>}</span>
−
         }
+
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>}</span>
 
    
 
    
−
         return $this->directoryCache[$path];
+
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>return $this->directoryCache[$path];</span>
−
       }
+
       <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>}</span>
 
   
 
   
 
       /**
 
       /**
 
       * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
 
       * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
 
       * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
 
       * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
−
 
−
 
  
 
=== Code Block 6 ===
 
=== Code Block 6 ===
Line 93: Line 96:
 
  if (is_array($result['CommonPrefixes'])) {
 
  if (is_array($result['CommonPrefixes'])) {
 
     foreach ($result['CommonPrefixes'] as $prefix) {
 
     foreach ($result['CommonPrefixes'] as $prefix) {
−
         $files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));
+
         <span style='background-color: #fbe9eb; border: #f44256 4px solid;'>$files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$directoryName = trim($prefix['Prefix'], '/');</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$directoryName = trim($prefix['Prefix'], '/');</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$files[] = substr($directoryName, strlen($path));</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>$files[] = substr($directoryName, strlen($path));</span>
Line 110: Line 113:
 
   
 
   
 
     try {
 
     try {
−
         return $this->isRoot($path) || $this->headObject($path . '/');
+
         <span style='background-color: #fbe9eb; border: #f44256 4px solid;'>return $this->isRoot($path) || $this->headObject($path . '/');</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>return $this->isRoot($path) || $this->doesDirectoryExist($path);</span>
 
         <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>return $this->isRoot($path) || $this->doesDirectoryExist($path);</span>
 
     } catch (S3Exception $e) {
 
     } catch (S3Exception $e) {
Line 121: Line 124:
 
         return 'file';
 
         return 'file';
 
     }
 
     }
−
     if ($this->headObject($path . '/')) {
+
     <span style='background-color: #fbe9eb; border: #f44256 4px solid;'>if ($this->headObject($path . '/')) {</span>
−
     if ($this->doesDirectoryExist($path)) {
+
     <span style='background-color: #ecfdf0; border: #5bfb81 4px solid;'>if ($this->doesDirectoryExist($path)) {</span>
 
         return 'dir';
 
         return 'dir';
 
     }
 
     }
Line 128: Line 131:
  
  
 +
== Apply the Patch ==
  
−
 
+
Save the code changes, and commit to git.
−
Need to apply the patch.
 
  
 
  git add .
 
  git add .
−
  git commit -m "Apply S3 Patch".
+
  git commit -m "Apply S3 Patch."
  
 
== Return ==
 
== Return ==

Latest revision as of 23:52, 16 September 2019

Replace AmaonS3.php

You can find this file under:

server/apps/files_external/lib/Lib/Storage/AmazonS3.php

Manually Applying Code

  1. Bring up this file inside an editor.
  2. In each Block Section of Code:
    1. Navigate to the specified line number
    2. Remove any line of code highlighted in RED
    3. Add any new line of code highlighted in GREEN

Code Block 1

Navigate to Line 61:

/** @var CappedMemoryCache|Result[] */
private $objectCache;

/** @var CappedMemoryCache|bool[] */
private $directoryCache;

/** @var CappedMemoryCache|array */
private $filesCache;

Code Block 2

Navigate to Line 71:

parent::__construct($parameters);
$this->parseParams($parameters);
$this->objectCache = new CappedMemoryCache();
$this->directoryCache = new CappedMemoryCache();
$this->filesCache = new CappedMemoryCache();

Code Block 3

Navigate to Line 103:

private function clearCache() {
    $this->objectCache = new CappedMemoryCache();
    $this->directoryCache = new CappedMemoryCache();
    $this->filesCache = new CappedMemoryCache();
}

Code Block 4

Navigate to Line 115:

            unset($this->objectCache[$existingKey]);
        }
    }
    unset($this->filesCache[$key]);
    unset($this->directoryCache[$key], $this->filesCache[$key]);
}

Code Block 5

Navigate to Line 140:

        return $this->objectCache[$key];
    }

    /**
     * Return true if directory exists
     *
     * There are no folders in s3. A folder like structure could be archived
     * by prefixing files with the folder name.
     *
     * Implementation from flysystem-aws-s3-v3:
     * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
     *
     * @param $path
     * @return bool
     * @throws \Exception
     */
    private function doesDirectoryExist($path) {
        if (!isset($this->directoryCache[$path])) {
            // Maybe this isn't an actual key, but a prefix.
            // Do a prefix listing of objects to determine.
            try {
                $result = $this->getConnection()->listObjects([
                    'Bucket' => $this->bucket,
                    'Prefix' => rtrim($path, '/') . '/',
                    'MaxKeys' => 1,
                ]);
                $this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
            } catch (S3Exception $e) {
                if ($e->getStatusCode() === 403) {
                    $this->directoryCache[$path] = false;
                }
                throw $e;
            }
        }
 
        return $this->directoryCache[$path];
     }

     /**
      * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
      * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home

Code Block 6

Navigate to Line 334:

// sub folders
if (is_array($result['CommonPrefixes'])) {
    foreach ($result['CommonPrefixes'] as $prefix) {
        $files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));
        $directoryName = trim($prefix['Prefix'], '/');
        $files[] = substr($directoryName, strlen($path));
        $this->directoryCache[$directoryName] = true;
    }
}

Code Block 7

Navigate to Line 434:

public function is_dir($path) {
    $path = $this->normalizePath($path);

    if (isset($this->filesCache[$path])) {
        return false;
    }

    try {
        return $this->isRoot($path) || $this->headObject($path . '/');
        return $this->isRoot($path) || $this->doesDirectoryExist($path);
    } catch (S3Exception $e) {
        \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
        return false;

Code Block 8

Navigate to Line 458:

    if (isset($this->filesCache[$path]) || $this->headObject($path)) {
        return 'file';
    }
    if ($this->headObject($path . '/')) {
    if ($this->doesDirectoryExist($path)) {
        return 'dir';
    }
} catch (S3Exception $e) {


Apply the Patch

Save the code changes, and commit to git.

git add .
git commit -m "Apply S3 Patch."

Return