Commit 01ad9c6d authored by Karandeep Bhatia's avatar Karandeep Bhatia Committed by Commit Bot

DocServer: Delete obsolete github file system code.

Delete code related to the github file system since it's not been used
for years. Also delete some other obsolete code.

BUG=820606, 1105728


Change-Id: Ia4b4e851ddd3b5ca640ca892bf27e3e934c96369
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2311897
Commit-Queue: Karan Bhatia <karandeepb@chromium.org>
Reviewed-by: default avatarDevlin <rdevlin.cronin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#791030}
parent 29eff5f9
......@@ -15,9 +15,8 @@ import sys
WHITELIST = [ r'.+_test.py$' ]
# The integration tests are selectively run from the PRESUBMIT in
# chrome/common/extensions. Github filesystem support is currently
# disabled.
BLACKLIST = [ r'integration_test.py$', r'.*github.*_test.py$' ]
# chrome/common/extensions.
BLACKLIST = [ r'integration_test.py$' ]
def _BuildServer(input_api):
try:
......
application: chrome-apps-doc
version: 3-70-0
version: 3-71-0
runtime: python27
api_version: 1
threadsafe: false
......
......@@ -7,8 +7,6 @@ from blob_reference_store import BlobReferenceStore
from appengine_wrappers import blobstore
from appengine_wrappers import files
BLOBSTORE_GITHUB = 'BlobstoreGithub'
# TODO(kalman): Re-write this class.
# - It uses BlobReader which is a synchronous method. We should be creating
# multiple async fetches, one for each partition, then exposing a Future
......
......@@ -121,15 +121,6 @@ class ContentProviders(object):
return None
file_system = ChrootFileSystem(self._host_file_system,
chromium_config['dir'])
# TODO(rockot): Remove this in a future patch. It should not be needed once
# the new content_providers.json is committed.
elif 'gitiles' in config:
chromium_config = config['gitiles']
if 'dir' not in chromium_config:
logging.error('%s: "chromium" must have a "dir" property' % name)
return None
file_system = ChrootFileSystem(self._host_file_system,
chromium_config['dir'])
elif 'gcs' in config:
gcs_config = config['gcs']
......
......@@ -28,22 +28,6 @@ _CONTENT_PROVIDERS = {
'dir': 'chrome/common/extensions'
},
},
'github-provider': {
'serveFrom': 'gh',
'github': {
'dir': 'chrome/common/extensions',
'owner': 'GoogleChrome',
'repo': 'hello-world',
},
},
'github-provider-with-dir': {
'serveFrom': 'gh2',
'github': {
'dir': 'chrome/common/extensions/tomatoes/are/a',
'owner': 'SomeOwner',
'repo': 'some-repo',
},
},
'tomatoes': {
'serveFrom': 'tomatoes-dir/are/a',
'chromium': {
......@@ -79,32 +63,11 @@ _FILE_SYSTEM_DATA = {
},
}
class _MockGithubFileSystemProvider(object):
'''A GithubFileSystemProvider imitation which records every call to Create
and returns them from GetAndReset.
'''
def __init__(self, file_system):
self._file_system = file_system
self._calls = []
def Create(self, owner, repo):
self._calls.append((owner, repo))
return self._file_system
def GetAndReset(self):
calls = self._calls
self._calls = []
return calls
class ContentProvidersTest(unittest.TestCase):
def setUp(self):
object_store_creator = ObjectStoreCreator.ForTest()
test_file_system = TestFileSystem(_FILE_SYSTEM_DATA,
relative_to=CHROME_EXTENSIONS)
self._github_fs_provider = _MockGithubFileSystemProvider(test_file_system)
object_store_creator = ObjectStoreCreator.ForTest()
# TODO(mangini): create tests for GCS
self._gcs_fs_provider = CloudStorageFileSystemProvider(object_store_creator)
......@@ -182,31 +145,5 @@ class ContentProvidersTest(unittest.TestCase):
def testProviderNotFound(self):
self.assertEqual(None, self._content_providers.GetByName('cabbages'))
# TODO: Re-enable these Github tests if we ever decide to restore our support
# for Github content providers.
# def testGithubContentProvider(self):
# provider, serve_from, path = self._content_providers.GetByServeFrom(
# 'gh/apples/green/granny smith.txt')
# self.assertEqual('github-provider', provider.name)
# self.assertEqual('gh', serve_from)
# self.assertEqual('apples/green/granny smith.txt', path)
# self.assertEqual([('GoogleChrome', 'hello-world')],
# self._github_fs_provider.GetAndReset())
# self.assertEqual(
# 'granny smith apples',
# provider.GetContentAndType(path).Get().content)
# def testGithubContentProviderWithDir(self):
# provider, serve_from, path = self._content_providers.GetByServeFrom(
# 'gh2/fruit/cherry.txt')
# self.assertEqual('github-provider-with-dir', provider.name)
# self.assertEqual('gh2', serve_from)
# self.assertEqual('fruit/cherry.txt', path)
# self.assertEqual([('SomeOwner', 'some-repo')],
# self._github_fs_provider.GetAndReset())
# self.assertEqual(
# 'cherry tomatoes',
# provider.GetContentAndType(path).Get().content)
if __name__ == '__main__':
unittest.main()
......@@ -106,17 +106,6 @@ class _FakeViewvcServer(_FakeFetcher):
except IOError:
return None
class _FakeGithubStat(_FakeFetcher):
def fetch(self, url):
return '{ "sha": 0 }'
class _FakeGithubZip(_FakeFetcher):
def fetch(self, url):
return _ReadTestData('github_file_system', 'apps_samples.zip', mode='rb')
class _FakeRietveldAPI(_FakeFetcher):
def __init__(self):
self._base_pattern = re.compile(r'.*/(api/.*)')
......@@ -144,8 +133,6 @@ def ConfigureFakeFetchers():
url_constants.OMAHA_PROXY_URL: _FakeOmahaProxy(),
'%s/.*' % url_constants.SVN_URL: _FakeSubversionServer(),
'%s/.*' % url_constants.VIEWVC_URL: _FakeViewvcServer(),
'%s/.*/commits/.*' % url_constants.GITHUB_REPOS: _FakeGithubStat(),
'%s/.*/zipball' % url_constants.GITHUB_REPOS: _FakeGithubZip(),
'%s/api/.*' % url_constants.CODEREVIEW_SERVER: _FakeRietveldAPI(),
'%s/tarball/.*' % url_constants.CODEREVIEW_SERVER: _FakeRietveldTarball(),
})
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
from StringIO import StringIO
import posixpath
from appengine_blobstore import AppEngineBlobstore, BLOBSTORE_GITHUB
from appengine_url_fetcher import AppEngineUrlFetcher
from appengine_wrappers import urlfetch, blobstore
from docs_server_utils import StringIdentity
from file_system import FileSystem, StatInfo
from future import Future
from path_util import IsDirectory
import url_constants
from zipfile import ZipFile, BadZipfile
ZIP_KEY = 'zipball'
USERNAME = None
PASSWORD = None
def _MakeBlobstoreKey(version):
return ZIP_KEY + '.' + str(version)
def _GetAsyncFetchCallback(fetcher,
username,
password,
blobstore,
key_to_set,
key_to_delete=None):
fetch = fetcher.FetchAsync(ZIP_KEY, username=username, password=password)
def resolve():
try:
result = fetch.Get()
# Check if Github authentication failed.
if result.status_code == 401:
logging.error('Github authentication failed for %s, falling back to '
'unauthenticated.' % USERNAME)
blob = fetcher.Fetch(ZIP_KEY).content
else:
blob = result.content
except urlfetch.DownloadError as e:
logging.error('Bad github zip file: %s' % e)
return None
if key_to_delete is not None:
blobstore.Delete(_MakeBlobstoreKey(key_to_delete, BLOBSTORE_GITHUB))
try:
return_zip = ZipFile(StringIO(blob))
except BadZipfile as e:
logging.error('Bad github zip file: %s' % e)
return None
blobstore.Set(_MakeBlobstoreKey(key_to_set), blob, BLOBSTORE_GITHUB)
return return_zip
return resolve
class GithubFileSystem(FileSystem):
@staticmethod
def CreateChromeAppsSamples(object_store_creator):
return GithubFileSystem(
'%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS,
AppEngineBlobstore(),
object_store_creator)
def __init__(self, url, blobstore, object_store_creator):
# If we key the password store on the app version then the whole advantage
# of having it in the first place is greatly lessened (likewise it should
# always start populated).
password_store = object_store_creator.Create(
GithubFileSystem,
app_version=None,
category='password',
start_empty=False)
if USERNAME is None:
password_data = password_store.GetMulti(('username', 'password')).Get()
self._username, self._password = (password_data.get('username'),
password_data.get('password'))
else:
password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
self._username, self._password = (USERNAME, PASSWORD)
self._url = url
self._fetcher = AppEngineUrlFetcher(url)
self._blobstore = blobstore
self._stat_object_store = object_store_creator.Create(GithubFileSystem)
self._version = None
self._GetZip(self.Stat(ZIP_KEY).version)
def _GetZip(self, version):
try:
blob = self._blobstore.Get(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
except blobstore.BlobNotFoundError:
self._zip_file = Future(value=None)
return
if blob is not None:
try:
self._zip_file = Future(value=ZipFile(StringIO(blob)))
except BadZipfile as e:
self._blobstore.Delete(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
logging.error('Bad github zip file: %s' % e)
self._zip_file = Future(value=None)
else:
self._zip_file = Future(
callback=_GetAsyncFetchCallback(self._fetcher,
self._username,
self._password,
self._blobstore,
version,
key_to_delete=self._version))
self._version = version
def _ReadFile(self, path):
try:
zip_file = self._zip_file.Get()
except Exception as e:
logging.error('Github ReadFile error: %s' % e)
return ''
if zip_file is None:
logging.error('Bad github zip file.')
return ''
prefix = zip_file.namelist()[0]
return zip_file.read(prefix + path)
def _ListDir(self, path):
try:
zip_file = self._zip_file.Get()
except Exception as e:
logging.error('Github ListDir error: %s' % e)
return []
if zip_file is None:
logging.error('Bad github zip file.')
return []
filenames = zip_file.namelist()
# Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
filenames = [f[len(filenames[0]):] for f in filenames]
# Remove the path of the directory we're listing from the filenames.
filenames = [f[len(path):] for f in filenames
if f != path and f.startswith(path)]
# Remove all files not directly in this directory.
return [f for f in filenames if f[:-1].count('/') == 0]
def Read(self, paths, skip_not_found=False):
version = self.Stat(ZIP_KEY).version
if version != self._version:
self._GetZip(version)
result = {}
for path in paths:
if IsDirectory(path):
result[path] = self._ListDir(path)
else:
result[path] = self._ReadFile(path)
return Future(value=result)
def _DefaultStat(self, path):
version = 0
# TODO(kalman): we should replace all of this by wrapping the
# GithubFileSystem in a CachingFileSystem. A lot of work has been put into
# CFS to be robust, and GFS is missing out.
# For example: the following line is wrong, but it could be moot.
self._stat_object_store.Set(path, version)
return StatInfo(version)
def Stat(self, path):
version = self._stat_object_store.Get(path).Get()
if version is not None:
return StatInfo(version)
try:
result = self._fetcher.Fetch('commits/HEAD',
username=USERNAME,
password=PASSWORD)
except urlfetch.DownloadError as e:
logging.warning('GithubFileSystem Stat: %s' % e)
return self._DefaultStat(path)
# Check if Github authentication failed.
if result.status_code == 401:
logging.warning('Github authentication failed for %s, falling back to '
'unauthenticated.' % USERNAME)
try:
result = self._fetcher.Fetch('commits/HEAD')
except urlfetch.DownloadError as e:
logging.warning('GithubFileSystem Stat: %s' % e)
return self._DefaultStat(path)
# Parse response JSON - but sometimes github gives us invalid JSON.
try:
version = json.loads(result.content)['sha']
self._stat_object_store.Set(path, version)
return StatInfo(version)
except StandardError as e:
logging.warning(
('%s: got invalid or unexpected JSON from github. Response status ' +
'was %s, content %s') % (e, result.status_code, result.content))
return self._DefaultStat(path)
def GetIdentity(self):
return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from caching_file_system import CachingFileSystem
from empty_dir_file_system import EmptyDirFileSystem
from github_file_system import GithubFileSystem as OldGithubFileSystem
from new_github_file_system import GithubFileSystem as NewGithubFileSystem
class GithubFileSystemProvider(object):
'''Provides GithubFileSystems bound to an owner/repo pair.
'''
def __init__(self, object_store_creator):
self._object_store_creator = object_store_creator
def Create(self, owner, repo):
'''Creates a GithubFileSystem. For legacy reasons this is hacked
such that the apps samples returns the old GithubFileSystem.
|owner| is the owner of the GitHub account, e.g. 'GoogleChrome'.
|repo| is the repository name, e.g. 'devtools-docs'.
'''
if owner == 'GoogleChrome' and repo == 'chrome-app-samples':
# NOTE: The old GitHub file system implementation doesn't support being
# wrapped by a CachingFileSystem. It's also too slow to run on the dev
# server, since every app API page would need to read from it.
return OldGithubFileSystem.CreateChromeAppsSamples(
self._object_store_creator)
return CachingFileSystem(
NewGithubFileSystem.Create(owner, repo, self._object_store_creator),
self._object_store_creator)
@staticmethod
def ForEmpty():
class EmptyImpl(object):
def Create(self, owner, repo):
return EmptyDirFileSystem()
return EmptyImpl()
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import os
import sys
import unittest
from appengine_wrappers import files
from fake_fetchers import ConfigureFakeFetchers
from github_file_system import GithubFileSystem
from object_store_creator import ObjectStoreCreator
from test_util import Server2Path
class GithubFileSystemTest(unittest.TestCase):
def setUp(self):
ConfigureFakeFetchers()
self._base_path = Server2Path('test_data', 'github_file_system')
self._file_system = GithubFileSystem.CreateChromeAppsSamples(
ObjectStoreCreator.ForTest())
def _ReadLocalFile(self, filename):
with open(os.path.join(self._base_path, filename), 'r') as f:
return f.read()
def testList(self):
self.assertEqual(json.loads(self._ReadLocalFile('expected_list.json')),
self._file_system.Read(['']).Get())
def testRead(self):
self.assertEqual(self._ReadLocalFile('expected_read.txt'),
self._file_system.ReadSingle('analytics/launch.js').Get())
def testStat(self):
self.assertEqual(0, self._file_system.Stat('zipball').version)
def testKeyGeneration(self):
self.assertEqual(0, len(files.GetBlobKeys()))
self._file_system.ReadSingle('analytics/launch.js').Get()
self.assertEqual(1, len(files.GetBlobKeys()))
self._file_system.ReadSingle('analytics/main.css').Get()
self.assertEqual(1, len(files.GetBlobKeys()))
if __name__ == '__main__':
unittest.main()
......@@ -66,9 +66,6 @@ class InstanceServlet(object):
def CreateHostFileSystemProvider(self, object_store_creator, **optargs):
return HostFileSystemProvider(object_store_creator, **optargs)
def CreateGithubFileSystemProvider(self, object_store_creator):
return GithubFileSystemProvider(object_store_creator)
@staticmethod
def GetConstructor(delegate_for_test=None):
render_servlet_delegate = InstanceServletRenderServletDelegate(
......
......@@ -20,9 +20,6 @@ class _TestDelegate(InstanceServlet.Delegate):
def CreateBranchUtility(self, object_store_creator):
return TestBranchUtility.CreateWithCannedData()
def CreateGithubFileSystemProvider(self, object_store_creator):
return GithubFileSystemProvider.ForEmpty()
class InstanceServletTest(unittest.TestCase):
'''Tests that if the file systems underlying the docserver's data fail,
the instance servlet still returns 404s or 301s with a best-effort.
......
......@@ -48,10 +48,6 @@ class _PatchServletDelegate(RenderServlet.Delegate):
def CreateHostFileSystemProvider(self, object_store_creator, **optargs):
return HostFileSystemProvider.ForLocal(object_store_creator, **optargs)
def CreateGithubFileSystemProvider(self, object_store_creator):
return GithubFileSystemProvider.ForEmpty()
class PatchServletTest(unittest.TestCase):
def setUp(self):
ConfigureFakeFetchers()
......
......@@ -55,8 +55,6 @@ class PlatformBundle(object):
extension_samples_fs = self._host_fs_at_master
# TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
app_samples_fs = EmptyDirFileSystem()
#app_samples_fs = github_file_system_provider.Create(
# 'GoogleChrome', 'chrome-app-samples')
self._platform_data[platform].samples_model = SamplesModel(
extension_samples_fs,
app_samples_fs,
......
......@@ -14,20 +14,3 @@ branch_utility/second.json
- Contains a history of revision names for the dev channel on Windows.
- File has been formatted for readability. To view a readable version of the
data at the above URL, remove '&json=1'.
subversion_file_system/public_extensions_214898
Downloaded from:
http://src.chromium.org/viewvc/chrome/trunk/src/chrome/common/extensions/docs/templates/public/extensions/?pathrev=214898
- Subversion ViewVC list of files in c/c/e/d/templates/public/extensions where
the most recent operation in that directory is deleting a file.
- Used in SubversionFileSystemTest.testDirectoryVersionOnDeletion.
github_file_system/test_owner
Created to mock a typical Python project on github.com
# TODO(jshumway): Use real data if it ever becomes a problem.
- Used in the github_file_system tests.
gitiles_file_system/public_extensions
Downloaded from:
https://chromium.googlesource.com/chromium/src/+/master/chrome/common/extensions/docs/templates/public/extensions/?format=JSON
- Gitiles JSON view of the files in c/c/e/d/templates/public/extensions
{
"": [
".gitignore",
"README.md",
"analytics/",
"appsquare/",
"browser-tag/",
"calculator/",
"camera-capture/",
"clock/",
"context-menu/",
"diff/",
"dojo/",
"filesystem-access/",
"frameless-window/",
"gdocs/",
"hello-world/",
"identity/",
"io2012-presentation/",
"ioio/",
"mdns-browser/",
"mini-code-edit/",
"nodejs-net.coffee",
"sandbox/",
"sandboxed-content/",
"serial-control-signals/",
"serial/",
"servo/",
"singleton/",
"storage/",
"telnet/",
"text-editor/",
"udp/",
"usb/",
"weather/",
"webgl/",
"webintents/",
"windows/",
"zephyr_hxm/"
]
}
/**
* Listens for the app launching then creates the window
*
* @see http://developer.chrome.com/trunk/apps/app.runtime.html
* @see http://developer.chrome.com/trunk/apps/app.window.html
*/
chrome.app.runtime.onLaunched.addListener(function() {
chrome.app.window.create('main.html',
{width: 480, height: 225});
});
CairoSVG==0.5
Markdown==2.3.1
Pyphen==0.7
WeasyPrint==0.19.1
argparse==1.2.1
cairocffi==0.4.2
cffi==0.6
cssselect==0.8
distribute==0.6.24
lxml==3.2.1
pep8==1.4.5
pycparser==2.09.1
pyflakes==0.7.2
tinycss==0.3
wsgiref==0.1.2
#!/usr/bin/env python
import random
def main(argv, argc):
if argc == 0:
print 'Hello, world!'
print 'usage: hello.py [names]'
else:
print 'Hello, %s!' % random.choice(argv[1:])
if __name__ == '__main__':
import sys
main(sys.argv, len(sys.argv))
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import getpass
import os
import subprocess
import sys
import build_server
if __name__ == '__main__':
additional_args = []
if len(sys.argv) > 1 and sys.argv[1].endswith('appcfg.py'):
appcfg_path = sys.argv[1]
additional_args = sys.argv[2:]
else:
appcfg_path = None
additional_args = sys.argv[1:]
for path in ['.',
os.path.join(os.environ['HOME'], 'local', 'google_appengine'),
os.path.join(os.environ['HOME'], 'google_appengine'),
os.getcwd()] + sys.path:
full_path = os.path.join(path, 'appcfg.py')
if os.path.exists(full_path):
appcfg_path = full_path
break
if appcfg_path is None:
print 'appcfg.py could not be found in default paths.'
print 'usage: update_server.py <path_to_appcfg.py> <appcfg_options>'
exit(1)
def run_appcfg():
server2_path = os.path.dirname(sys.argv[0])
subprocess.call([appcfg_path, 'update', server2_path] + additional_args)
build_server.main()
username = raw_input(
'Update github username/password (empty to skip)? ')
if username:
password = getpass.getpass()
with open('github_file_system.py') as f:
contents = f.read()
if 'USERNAME = None' not in contents:
print 'Error: Can\'t find "USERNAME = None" in github_file_system.py.'
exit(1)
if 'PASSWORD = None' not in contents:
print 'Error: Can\'t find "PASSWORD = None" in github_file_system.py.'
exit(1)
try:
with open('github_file_system.py', 'w+') as f:
f.write(
contents.replace('PASSWORD = None', 'PASSWORD = \'%s\'' % password)
.replace('USERNAME = None', 'USERNAME = \'%s\'' % username))
run_appcfg()
finally:
with open('github_file_system.py', 'w+') as f:
f.write(contents)
else:
run_appcfg()
......@@ -9,13 +9,10 @@
// * An arbitrary identifier key e.g. "cr-extensions-examples".
// * What URL the rule should be invoked with, given by "serveFrom", e.g.
// "extensions/examples".
// * An object describing where the content originates, either "chromium",
// "github", or "gcs".
// * An object describing where the content originates, either "chromium" or
// * "gcs".
// * "chromium" must provide a "dir" value specifying which chromium directory
// to look in, e.g. "extensions/samples".
// * "github" must provide "owner" and "repo" values specifying the owner of
// the GitHub repository, and the repository name, e.g. "GoogleChrome" and
// "chrome-app-samples" respectively.
//
// In the chromium example, when the user navigates to
//
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment