Import from Filesystem
======================

Give test user the Manager role and login:

  >>> from Testing.ZopeTestCase import user_name, user_password, folder_name
  >>> self.setRoles(['Manager'])
  >>> self.login(user_name)
  >>> request = self.portal.REQUEST

Create some temporary files, to make sure we have something to point to:

  >>> import os
  >>> import tempfile
  >>> tmp_dir = tempfile.mkdtemp()
  >>> j = os.path.join
  >>> def mkdir(*args):
  ...   os.mkdir(j(*args))
  >>> def touch(*args):
  ...   open(j(*args), 'wb+').close()
  >>> mkdir(tmp_dir, 'dir1')
  >>> mkdir(tmp_dir, 'dir1', 'dir11')
  >>> touch(tmp_dir, 'dir1', 'dir11', 'dir11_file.txt')
  >>> touch(tmp_dir, 'dir1', 'dir11', 'dir11_file.rtf')
  >>> touch(tmp_dir, 'dir1', 'dir11', 'dir11_file.pdf')
  >>> mkdir(tmp_dir, 'dir2')
  >>> touch(tmp_dir, 'dir2', 'dir2_index.html')
  >>> mkdir(tmp_dir, 'dir2', 'dir21')
  >>> touch(tmp_dir, 'dir2', 'dir21', 'dir21_file.bat')
  >>> mkdir(tmp_dir, 'dir2', 'dir22')
  >>> mkdir(tmp_dir, 'dir2', 'dir22', 'dir221')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir221', 'file%221.exe')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir221', 'file-ao.exe')

Here we are creating a file that will end up as a duplicate when
importing because of the filename cleaning that goes on:

  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir221', 'file@221.exe')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir221', 'file!.doc')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir221', 'file@221.xls')
  >>> mkdir(tmp_dir, 'dir2', 'dir22', 'dir222')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir222', 'file.tar.gz')
  >>> touch(tmp_dir, 'dir2', 'dir22', 'dir222', 'file.xml')

Set the default repository path to point to the tmp_dir:

  >>> from Products.CMFManagedFile.config import *
  >>> self.tool.getRepository('default').edit(path=tmp_dir,
  ...                                         path_policy=FS_BASE)


Create a folder for importing, use simplest import possible:

  >>> self.portal.invokeFactory('Folder', 'folder1')
  >>> load = self.tool.bulkImportFilesystemPath
  >>> res = load(self.portal.folder1, recursive=True)

  >>> count(self.portal.folder1, PortalTypeCounter)
  Folder: 7
  ManagedFile: 11
  Total: 18

  >>> count(self.portal.folder1, RelativeURLCounter)
  folder1/dir1: 1
  folder1/dir1/dir11: 1
  folder1/dir1/dir11/dir11_file.pdf: 1
  folder1/dir1/dir11/dir11_file.rtf: 1
  folder1/dir1/dir11/dir11_file.txt: 1
  folder1/dir2: 1
  folder1/dir2/dir21: 1
  folder1/dir2/dir21/dir21_file.bat: 1
  folder1/dir2/dir22: 1
  folder1/dir2/dir22/dir221: 1
  folder1/dir2/dir22/dir221/file-a__o.exe: 1
  folder1/dir2/dir22/dir221/file.doc: 1
  folder1/dir2/dir22/dir221/file221.exe: 1
  folder1/dir2/dir22/dir221/file221.xls: 1
  folder1/dir2/dir22/dir222: 1
  folder1/dir2/dir22/dir222/file.tar.gz: 1
  folder1/dir2/dir22/dir222/file.xml: 1
  folder1/dir2/dir2_index.html: 1
  Total: 18

  >>> count(self.portal.folder1, PathExistsCounter)
  Non-Verifiable: 7
  True: 11
  Total: 18

Test suppressing directory names:

  >>> self.portal.invokeFactory('Folder', 'folder2')
  >>> res = load(self.portal.folder2, recursive=True,
  ...            dirNames='dir221 dir1')

  >>> count(self.portal.folder2, PortalTypeCounter)
  Folder: 4
  ManagedFile: 4
  Total: 8

  >>> count(self.portal.folder2, RelativeURLCounter)
  folder2/dir2: 1
  folder2/dir2/dir21: 1
  folder2/dir2/dir21/dir21_file.bat: 1
  folder2/dir2/dir22: 1
  folder2/dir2/dir22/dir222: 1
  folder2/dir2/dir22/dir222/file.tar.gz: 1
  folder2/dir2/dir22/dir222/file.xml: 1
  folder2/dir2/dir2_index.html: 1
  Total: 8

  >>> count(self.portal.folder2, PathExistsCounter)
  Non-Verifiable: 4
  True: 4
  Total: 8

Test suppressing directory prefixes:

  >>> self.portal.invokeFactory('Folder', 'folder3')
  >>> res = load(self.portal.folder3, recursive=True,
  ...            dirPrefixes='dir221 dir11')

  >>> count(self.portal.folder3, PortalTypeCounter)
  Folder: 5
  ManagedFile: 4
  Total: 9

  >>> count(self.portal.folder3, RelativeURLCounter)
  folder3/dir1: 1
  folder3/dir2: 1
  folder3/dir2/dir21: 1
  folder3/dir2/dir21/dir21_file.bat: 1
  folder3/dir2/dir22: 1
  folder3/dir2/dir22/dir222: 1
  folder3/dir2/dir22/dir222/file.tar.gz: 1
  folder3/dir2/dir22/dir222/file.xml: 1
  folder3/dir2/dir2_index.html: 1
  Total: 9

  >>> count(self.portal.folder3, PathExistsCounter)
  Non-Verifiable: 5
  True: 4
  Total: 9

Test suppressing directory suffixes:

  >>> self.portal.invokeFactory('Folder', 'folder4')
  >>> res = load(self.portal.folder4, recursive=True,
  ...            dirSuffixes='22')

  >>> count(self.portal.folder4, PortalTypeCounter)
  Folder: 4
  ManagedFile: 5
  Total: 9

  >>> count(self.portal.folder4, RelativeURLCounter)
  folder4/dir1: 1
  folder4/dir1/dir11: 1
  folder4/dir1/dir11/dir11_file.pdf: 1
  folder4/dir1/dir11/dir11_file.rtf: 1
  folder4/dir1/dir11/dir11_file.txt: 1
  folder4/dir2: 1
  folder4/dir2/dir21: 1
  folder4/dir2/dir21/dir21_file.bat: 1
  folder4/dir2/dir2_index.html: 1
  Total: 9

  >>> count(self.portal.folder4, PathExistsCounter)
  Non-Verifiable: 4
  True: 5
  Total: 9

Test omitting filenames:

  >>> self.portal.invokeFactory('Folder', 'folder5')
  >>> res = load(self.portal.folder5, recursive=True,
  ...            fileNames='dir11_file.pdf dir21_file.bat dir2_index.html')

  >>> count(self.portal.folder5, PortalTypeCounter)
  Folder: 7
  ManagedFile: 8
  Total: 15

  >>> count(self.portal.folder5, RelativeURLCounter)
  folder5/dir1: 1
  folder5/dir1/dir11: 1
  folder5/dir1/dir11/dir11_file.rtf: 1
  folder5/dir1/dir11/dir11_file.txt: 1
  folder5/dir2: 1
  folder5/dir2/dir21: 1
  folder5/dir2/dir22: 1
  folder5/dir2/dir22/dir221: 1
  folder5/dir2/dir22/dir221/file-a__o.exe: 1
  folder5/dir2/dir22/dir221/file.doc: 1
  folder5/dir2/dir22/dir221/file221.exe: 1
  folder5/dir2/dir22/dir221/file221.xls: 1
  folder5/dir2/dir22/dir222: 1
  folder5/dir2/dir22/dir222/file.tar.gz: 1
  folder5/dir2/dir22/dir222/file.xml: 1
  Total: 15

  >>> count(self.portal.folder5, PathExistsCounter)
  Non-Verifiable: 7
  True: 8
  Total: 15

Test omitting file prefixes:

  >>> self.portal.invokeFactory('Folder', 'folder6')
  >>> res = load(self.portal.folder6, recursive=True,
  ...            filePrefixes='dir')

  >>> count(self.portal.folder6, PortalTypeCounter)
  Folder: 7
  ManagedFile: 6
  Total: 13

  >>> count(self.portal.folder6, RelativeURLCounter)
  folder6/dir1: 1
  folder6/dir1/dir11: 1
  folder6/dir2: 1
  folder6/dir2/dir21: 1
  folder6/dir2/dir22: 1
  folder6/dir2/dir22/dir221: 1
  folder6/dir2/dir22/dir221/file-a__o.exe: 1
  folder6/dir2/dir22/dir221/file.doc: 1
  folder6/dir2/dir22/dir221/file221.exe: 1
  folder6/dir2/dir22/dir221/file221.xls: 1
  folder6/dir2/dir22/dir222: 1
  folder6/dir2/dir22/dir222/file.tar.gz: 1
  folder6/dir2/dir22/dir222/file.xml: 1
  Total: 13

  >>> count(self.portal.folder6, PathExistsCounter)
  Non-Verifiable: 7
  True: 6
  Total: 13

Test omitting file suffixes:

  >>> self.portal.invokeFactory('Folder', 'folder7')
  >>> res = load(self.portal.folder7, recursive=True,
  ...            fileSuffixes='pdf bat html .exe')

  >>> count(self.portal.folder7, PortalTypeCounter)
  Folder: 7
  ManagedFile: 6
  Total: 13

  >>> count(self.portal.folder7, RelativeURLCounter)
  folder7/dir1: 1
  folder7/dir1/dir11: 1
  folder7/dir1/dir11/dir11_file.rtf: 1
  folder7/dir1/dir11/dir11_file.txt: 1
  folder7/dir2: 1
  folder7/dir2/dir21: 1
  folder7/dir2/dir22: 1
  folder7/dir2/dir22/dir221: 1
  folder7/dir2/dir22/dir221/file.doc: 1
  folder7/dir2/dir22/dir221/file221.xls: 1
  folder7/dir2/dir22/dir222: 1
  folder7/dir2/dir22/dir222/file.tar.gz: 1
  folder7/dir2/dir22/dir222/file.xml: 1
  Total: 13

  >>> count(self.portal.folder7, PathExistsCounter)
  Non-Verifiable: 7
  True: 6
  Total: 13

Test target directory:

  >>> self.portal.invokeFactory('Folder', 'folder8')
  >>> res = load(self.portal.folder8, recursive=True,
  ...            target_dir='dir2/dir22')

  >>> count(self.portal.folder8, PortalTypeCounter)
  Folder: 2
  ManagedFile: 6
  Total: 8

  >>> count(self.portal.folder8, RelativeURLCounter)
  folder8/dir221: 1
  folder8/dir221/file-a__o.exe: 1
  folder8/dir221/file.doc: 1
  folder8/dir221/file221.exe: 1
  folder8/dir221/file221.xls: 1
  folder8/dir222: 1
  folder8/dir222/file.tar.gz: 1
  folder8/dir222/file.xml: 1
  Total: 8

  >>> count(self.portal.folder8, PathExistsCounter)
  Non-Verifiable: 2
  True: 6
  Total: 8

Test non-recursive & target directory:

  >>> self.portal.invokeFactory('Folder', 'folder9')
  >>> res = load(self.portal.folder9, recursive=False,
  ...            target_dir='dir1/dir11')

  >>> count(self.portal.folder9, PortalTypeCounter)
  ManagedFile: 3
  Total: 3

  >>> count(self.portal.folder9, RelativeURLCounter)
  folder9/dir11_file.pdf: 1
  folder9/dir11_file.rtf: 1
  folder9/dir11_file.txt: 1
  Total: 3

  >>> count(self.portal.folder9, PathExistsCounter)
  True: 3
  Total: 3


Make directory unreadable, test recursive & target directory and
verify the output:

  >>> self.portal.invokeFactory('Folder', 'folder10')
  >>> os.chmod(os.path.join(tmp_dir, 'dir1', 'dir11'), 0100)
  >>> res = load(self.portal.folder10, recursive=True,
  ...            target_dir='dir1')

No exception should happen, but the contained files will not be
imported because the user doesn't have permissions. This is an
implementation detail of os.path.walk.

  >>> print res
  Processing...
  Created Folder: dir11 from directory...

  >>> count(self.portal.folder10, PortalTypeCounter)
  Folder: 1
  Total: 1

  >>> count(self.portal.folder10, RelativeURLCounter)
  folder10/dir11: 1
  Total: 1

  >>> count(self.portal.folder10, PathExistsCounter)
  Non-Verifiable: 1
  Total: 1

  >>> os.chmod(os.path.join(tmp_dir, 'dir1', 'dir11'), 0700)

Make a file unreadable, test recursive & target directory and
verify the output:

  >>> self.portal.invokeFactory('Folder', 'folder11')
  >>> os.chmod(os.path.join(tmp_dir, 'dir1', 'dir11', 'dir11_file.txt'), 0)
  >>> res = load(self.portal.folder11, recursive=True,
  ...            target_dir='dir1')

  >>> print res
  Processing...
  ...
  Skipping file: ...dir11_file.txt...

  >>> count(self.portal.folder11, PortalTypeCounter)
  Folder: 1
  ManagedFile: 2
  Total: 3

  >>> count(self.portal.folder11, RelativeURLCounter)
  folder11/dir11: 1
  folder11/dir11/dir11_file.pdf: 1
  folder11/dir11/dir11_file.rtf: 1
  Total: 3

  >>> count(self.portal.folder11, PathExistsCounter)
  Non-Verifiable: 1
  True: 2
  Total: 3

  >>> os.chmod(os.path.join(tmp_dir, 'dir1', 'dir11', 'dir11_file.txt'), 0700)

Cleanup the temp directory:

  >>> import shutil
  >>> shutil.rmtree(tmp_dir)
