Interim synchronization with internal repository

This is an intermir commit to share what is going on with the development of the new /delta version.

scripts/builder/backendBuilder.py

@@ -92,6 +92,7 @@ class BackendBuilder(object):
 		print message + ": " + md5Digest + " (md5)"
 		print message + ": " + shaDigest + " (sha1)"
 		print message + ": " + sha256Digest + " (sha256)"
+		print "file size: " + "{:,}".format(len(content))
 		print "====="
 		
 	
@@ -112,14 +113,15 @@ class BackendBuilder(object):
 					submoduleExtension = '.' + frontend.submodule
 
 				main.createFolder(os.path.join(self.frontEndTempFolder(), frontend.module))
+				frontend.copyResourcesToFolder(self.frontEndTempFolder())
 
 				if 'debug' in self.versions:
-					frontend.copyResourcesToFolder(self.frontEndTempFolder())
-
+					frontend.copyDebugResourcesToFolder(self.frontEndTempFolder())
 					index = self.configureIndexContent(frontend.assemble(assemblyMode='DEBUG', versionType='DEBUG'))
 					self.writeToFolder(self.frontEndTempFolder(), os.path.join(frontend.module, 'index_debug' + submoduleExtension + '.html'), index)
 
 				if 'install' in self.versions:
+#					frontend.copyResourcesToFolder(self.frontEndTempFolder())
 					index = self.configureIndexContent(frontend.assemble())
 					self.writeToFolder(self.frontEndTempFolder(), os.path.join(frontend.module, 'index' + submoduleExtension + '.html'), index)
 

scripts/builder/backends/checksumBuilder.py

@@ -42,6 +42,7 @@ class ChecksumBuilder(BackendBuilder):
 
 			index = self.configureIndexContent(frontend.assemble())
 			self.writeToFolder(self.frontEndTempFolder(), os.path.join(frontend.module, 'index' + submoduleExtension + '.html'), index)
+			frontend.copyResourcesToFolder(self.frontEndTempFolder())
 			self.logChecksums(index, "[" + self.name() + " - " + frontend.module + "] index" + submoduleExtension + ".html checksum")
 			print ""
 

scripts/builder/backends/devBuilder.py

@@ -39,6 +39,7 @@ class DevBuilder(BackendBuilder):
 				submoduleExtension = '.' + frontend.submodule
 
 			main.createFolder(os.path.join(self.frontEndTempFolder(), frontend.module))
+			frontend.copyResourcesToFolder(self.frontEndTempFolder())
 
 			index = self.configureIndexContent(frontend.assemble(assemblyMode='DEBUG', versionType='DEBUG'))
 			self.writeToFolder(self.frontEndTempFolder(), os.path.join(frontend.module, 'index' + submoduleExtension + '.html'), index)

scripts/builder/frontendBuilder.py

@@ -38,9 +38,15 @@ class FrontendBuilder(object):
 	def projectResourceTypes (self):
 		raise NotImplementedError()
 
-	def copyStaticResources (self, targetFolder):
+#	def copyStaticResources (self, targetFolder):
+#		raise NotImplementedError()
+
+	def copyResourcesToFolder (self, targetFolder):
 		raise NotImplementedError()
 
+	def preprocessCSS (self, targetFile):
+		pass
+
 	#---------------------------------------------------------------------------
 
 	def log (self, message):
@@ -72,7 +78,7 @@ class FrontendBuilder(object):
 				result.append(file)
 			
 		return result
-	
+
 
 	def copyResources (self, sourceFolder, destinationFolder, fileType):
 		if fileType in self.settings:
@@ -94,17 +100,18 @@ class FrontendBuilder(object):
 
 		
 
-	def copyResourcesToFolder (self, targetFolder):
-#		self.copyResources(self.projectDir, targetFolder, 'css')
-#		self.copyResources(self.projectDir, targetFolder, 'js')
-#		self.copyResources(self.projectDir, targetFolder, 'images')
+#	def copyResourcesToFolder (self, targetFolder):
+#		for resoureceType in self.projectResourceTypes():
+#			self.copyResources(self.projectDir, targetFolder, resoureceType)
+#		self.copyStaticResources(targetFolder)
+	
+	def copyDebugResourcesToFolder (self, targetFolder):
 		for resoureceType in self.projectResourceTypes():
 			self.copyResources(self.projectDir, targetFolder, resoureceType)
-		self.copyStaticResources(targetFolder)
-	
 
-	def loadFilesContent (self, basePath, files):
-		result = ""
+
+	def loadIndividualFilesContent (self, basePath, files):
+		result = {}
 		
 		for file in self.filterFiles(files):
 			try:
@@ -112,11 +119,39 @@ class FrontendBuilder(object):
 			except:
 				print "FILE: " + file
 
-			result += fileHandler.read() + '\n'
+			result[file] = fileHandler.read()
 			fileHandler.close()
-			
+		
 		return result
-	
+
+
+	def loadFilesContent (self, basePath, files):
+		result = ""
+		
+#		for file in self.filterFiles(files):
+#			try:
+#				fileHandler = codecs.open(self.absolutePathForSourceFile(basePath, file), 'r', 'utf-8')
+#			except:
+#				print "FILE: " + file
+#
+#			result += fileHandler.read() + '\n'
+#			fileHandler.close()
+
+#		for name, content in self.loadIndividualFilesContent(basePath, files):
+		for name, content in list(self.loadIndividualFilesContent(basePath, files).items()):
+			result += content + '\n'
+		
+		return result
+
+
+#	def packFilesContent (self, filesContent):
+#		result = ""
+#		
+#		for name, content in filesContent:
+#			result += content + '\n'
+#			
+#		return result
+
 
 	def template (self):
 		processedFile = 'html_template'
@@ -127,6 +162,8 @@ class FrontendBuilder(object):
 		return self.processedFiles[processedFile]
 	
 
+	#==========================================================================
+
 	def cssminCompressor (self, css):
 		# package found here:
 		# - http://stackoverflow.com/questions/222581/python-script-for-minifying-css/2396777#2396777
@@ -172,13 +209,14 @@ class FrontendBuilder(object):
 		        print "%s{%s}" % ( ','.join( selectors ), ''.join(['%s:%s;' % (key, properties[key]) for key in porder])[:-1] )
 		
 		return css
-	
+
 
 	def compressCSS (self, css):
 		self.log("compressing CSS")
 		#return self.regexCssCompressor(css)
 		return self.cssminCompressor(css)
-	
+
+	#--------------------------------------------------------------------------
 
 	#==========================================================================
 
@@ -262,6 +300,8 @@ class FrontendBuilder(object):
 		return result
 	
 
+	#==========================================================================
+
 	def replaceTemplatePlaceholders (self, pageTitle, copyright, css, code, jsLoadMode, version, versionType):
 		result = self.template()
 		
@@ -332,6 +372,9 @@ class FrontendBuilder(object):
 		else:
 			pageTitle = "Clipperz - " + self.module + " [" + versionType + " - " + assemblyMode +"]"
 
+		for cssFile in self.settings['css']:
+			self.preprocessCSS(self.absolutePathForSourceFile('css', cssFile))
+
 		if assemblyMode == 'INSTALL':
 			copyright = self.assembleCopyrightHeader()
 			css	=	self.cssTagForContent(self.compressCSS(self.loadFilesContent('css', self.settings['css'])))

scripts/builder/frontends/betaBuilder.py

@@ -9,7 +9,10 @@ class BetaBuilder(FrontendBuilder):
 	def projectResourceTypes (self):
 		return ['js', 'css', 'images']
 
-	def copyStaticResources (self, targetFolder):
+#	def copyStaticResources (self, targetFolder):
+	def copyResourcesToFolder (self, targetFolder):
+		self.copyResources(self.projectDir, targetFolder, 'images')
+
 		for file in self.filterFiles(self.settings['staticResources']):
 			src = self.absolutePathForSourceFile('staticResources', file)
 			dst = self.absolutePathForTargetFile(targetFolder, '', file)

scripts/builder/frontends/deltaBuilder.py

@@ -1,5 +1,11 @@
 from frontendBuilder import FrontendBuilder
+from scss import Scss
+
+import os
 import shutil
+import main
+import logging
+
 
 class DeltaBuilder(FrontendBuilder):
 
@@ -9,15 +15,49 @@ class DeltaBuilder(FrontendBuilder):
 	def projectResourceTypes (self):
 		return ['js', 'css']
 
-	def copyStaticResources (self, targetFolder):
+
+#	def copyStaticResources (self, targetFolder):
+	def copyResourcesToFolder (self, targetFolder):
+		#print "DELTA - copyResourcesToFolder"
 		resourcesToCopy = [
 			{'folder': 'properties',	'source': 'manifest.appcache',	'target': 'manifest.appcache'}
 		]
 
 		for resource in resourcesToCopy:
-			src = self.absolutePathForSourceFile(resource['folder'], resource['source'])
+			#print "copying resource: " + str(resource['source'])
+			content = self.loadFilesContent(resource['folder'], [resource['source']])
+			content = content.replace('@application.version@',	self.repositoryVersion)
+
 			dst = self.absolutePathForTargetFile(targetFolder, '', resource['target'])
-			shutil.copy2(src, dst)
+			file = open(dst, 'w')
+			file.write(content.encode('utf-8'))
+			file.close()
+
 
 	def bookmarklet (self):
-		return ""
+		return ""
+
+	def preprocessCSS (self, targetFile):
+		logging.basicConfig()
+		scssVariables = {}
+		scssCompiler = Scss(
+			scss_vars = None,
+			scss_opts = {
+				'compress': True,
+#				'debug_info': True,
+			},
+			scss_files = self.loadIndividualFilesContent('scss', self.settings['scss']),
+#			super_selector = None,
+#			live_errors = False,
+#			library = ALL_BUILTINS_LIBRARY,
+			search_paths = [os.path.join(self.absolutePathForSources(), 'scss')]
+		)
+
+		cssFileContent = scssCompiler.compile()
+
+		dst = targetFile
+		dst = os.path.join(os.path.dirname(dst), "_" + os.path.basename(dst))
+		main.createFolder(os.path.dirname(dst))
+		file = open(dst, 'w')
+		file.write(cssFileContent.encode('utf-8'))
+		file.close()

scripts/builder/frontends/gammaBuilder.py

@@ -9,10 +9,13 @@ class GammaBuilder(FrontendBuilder):
 	def projectResourceTypes (self):
 		return ['js', 'css', 'images']
 
-	def copyStaticResources (self, targetFolder):
+#	def copyStaticResources (self, targetFolder):
+	def copyResourcesToFolder (self, targetFolder):
+		self.copyResources(self.projectDir, targetFolder, 'images')
+
 		resourcesToCopy = [
-#			{'folder': 'html',	'source': 'exit_template.html',	'target': 'exit.html'},
-			{'folder': 'html',	'source': 'exit_template.html',	'target': 'logout.html'},
+			{'folder': 'html',	'source': 'exit_template.html',	'target': 'exit.html'},
+#			{'folder': 'html',	'source': 'exit_template.html',	'target': 'logout.html'},
 			{'folder': 'css',	'source': 'static.css',			'target': 'static.css'}
 		]
 

scripts/builder/frontends/gamma_mobileBuilder.py

@@ -8,5 +8,6 @@ class Gamma_MobileBuilder(FrontendBuilder):
 	def projectResourceTypes (self):
 		return ['js', 'css', 'images']
 
-	def copyStaticResources (self, targetFolder):
+#	def copyStaticResources (self, targetFolder):
+	def copyResourcesToFolder (self, targetFolder):
 		pass

scripts/builder/main.py

@@ -89,6 +89,7 @@ def build (settings, repository):
 	
 	if repository.areTherePendingChanges():
 		if 'install' in settings['versions']:
+#			print "\nWARNING: repository has pending changes\n"
 			raise Exception("repository has pending changes, can't 'install'")
 		else:
 			print "\nWARNING: repository has pending changes\n"
@@ -135,7 +136,7 @@ def usage (message):
 #--------------------------------------------------------------------
 
 def allFrontends ():
-	return ['beta', 'gamma',  'gamma.mobile']
+	return ['beta', 'gamma',  'delta']
 
 def allBackends ():
 	return ['php',  'python']

scripts/builder/repository.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: UTF-8 -*-
 
-
 def repositoryWithPath (path):
 	try:
-		from mercurial import ui, hg
+		# pip install hgapi
+		import hgapi
 
-		repo = hg.repository(ui.ui(), path)
+		repo = hgapi.Repo(path)
 		result = HgRepository(repo, path)
+		repo.hg_status()
 	except:
 		try:
 			from git import Repo
@@ -17,6 +18,7 @@ def repositoryWithPath (path):
 			print "Failed to import git, please install http://gitorious.org/git-python"
 			print "Use sudo apt-get install python-git for Ubuntu/Debian"
 			print "Use sudo yum install GitPython for Fedora/RHEL/CentOS"
+			print "Or manually running the following command: easy_install gitpython"
 		except:
 			result = SnapshotRepository('', path)
 
@@ -47,7 +49,6 @@ class Repository(object):
 		if self.areTherePendingChanges():
 			result = '>>> ' + result + ' <<<'
 
-		# print "VERSION: " + result
 		return result
 
 
@@ -76,15 +77,14 @@ class GitRepository(Repository):
 
 
 class HgRepository(Repository):
-	#	http://mercurial.selenic.com/wiki/MercurialApi
+	##	http://mercurial.selenic.com/wiki/MercurialApi
+	#	https://bitbucket.org/haard/hgapi
 
 	def revision (self):
-		return 'hg:' + str(self.repository['tip'])
-	
+		return 'hg: ' + str(self.repository['tip'].node)
 
 	def areTherePendingChanges (self):
-		# TODO: FIXME: repository.status() does not report 'unknown(?)' files. :(
-		return not all(map(lambda fileList: len(fileList) == 0, self.repository.status()))
+		return not all(map(lambda fileList: len(fileList) == 0, self.repository.hg_status()))
 
 
 #===================================================================

scripts/proxy/main.py

@@ -1,11 +1,21 @@
-from twisted.internet import reactor
-from twisted.web import proxy, server, http, resource, static
-from posixpath import basename, dirname
+from twisted.internet	import reactor
+from twisted.web		import proxy, server, http, resource, static
+from posixpath			import basename, dirname
 
 import copy
 import sys
 import os
+import uuid
 import pprint
+import codecs
+import time
+import subprocess
+
+#! sys.path.append("/usr/local/lib/python2.7/site-packages")
+# > export PYTHONPATH=/usr/local/lib/python2.7/site-packages
+
+from watchdog.observers	import Observer
+from watchdog.events	import FileSystemEventHandler
 
 #--------------------------------------------------------------------
 
@@ -31,27 +41,36 @@ class ClipperzTestSite(server.Site):
 		uri = uri.split("?", 1)[0]
 		uri = uri.split("#", 1)[0]
 		if uri.startswith('/json') or uri.startswith('/dump'):
-			request.site = self
-			request.sitepath = copy.copy(request.prepath)
+			resource.prepath = ['app']
 			result = resource.getChildForRequest(self.resource, request)
-
+		elif uri.startswith('/payment'):
+			resource.prepath = ['payment']
+			result = resource.getChildForRequest(self.resource, request)
+		elif uri == '/favicon.ico':
+			return
 		else:
 			pathParts = uri.split('/')
 			version = pathParts[1]
 
 			if pathParts[2].startswith('index.'):
+				print("-> index")
 				contentType = 'text/html'
 				absoluteFilePath = os.path.join(projectTargetDir(), 'dev', version, pathParts[2])
+#				print("INDEX.HTML absolute path " + str(absoluteFilePath))
 				result = static.File(absoluteFilePath, contentType)
 			elif pathParts[2].endswith('.webapp'):
+				print("-> webapp")
 				contentType = 'application/x-web-app-manifest+json'
-#				absoluteFilePath = os.path.join(projectTargetDir(), 'dev', version, pathParts[2])
 				absoluteFilePath = os.path.join(projectBaseDir(), 'frontend', version, 'properties', pathParts[2])
 				result = static.File(absoluteFilePath, contentType)
 			elif pathParts[2].endswith('.appcache'):
+				print("-> appcache")
 				contentType = 'text/cache-manifest'
 				absoluteFilePath = os.path.join(projectBaseDir(), 'frontend', version, 'properties', pathParts[2])
-				result = static.File(absoluteFilePath, contentType)
+				fileContent = codecs.open(absoluteFilePath, 'r', 'utf-8').read()
+#				fileContent = fileContent.replace('@application.version@', str(uuid.uuid1()))
+				fileContent = fileContent.replace('@application.version@', str(round(time.time())))
+				result = static.Data(str(fileContent), contentType)
 			else: 
 #	http://homer.local:8888/beta/css/clipperz/images/loginInfoBackground.png
 #	pathParts: ['', 'beta', 'css', 'clipperz', 'images', 'loginInfoBackground.png']
@@ -67,10 +86,13 @@ class ClipperzTestSite(server.Site):
 				if resourceType == 'images':
 					fileExtension = os.path.splitext(uri)[1]
 					if fileExtension == '.png':
+#						print("-> image - png")
 						contentType = 'image/png'
 					elif fileExtension == '.jpg':
+#						print("-> image - jpg")
 						contentType = 'image/jpeg'
 					elif fileExtension == '.gif':
+#						print("-> image - gif")
 						contentType = 'image/gif'
 					else:
 						print "ERROR - unknown image extension: " + fileExtension
@@ -80,28 +102,47 @@ class ClipperzTestSite(server.Site):
 					resourceType = pathParts[2]
 
 					if resourceType == 'css':
+#						print("-> css")
 						contentType = 'text/css'
 					elif resourceType == 'js':
+#						print("-> js")
 						contentType = 'text/javascript'
 					else:
+#						print("-> text/html")
 						contentType = 'text/html'
 					
 					absoluteFilePath = basePath + uri
 
 				result = static.File(absoluteFilePath, contentType)
 
-
+#		print("RESULT\n" + str(result))
 		return result
 
 
+class BuildScriptRunner(FileSystemEventHandler):
+	def on_any_event(self, anEvent):
+		if (not anEvent.is_directory):
+#			if (anEvent.src_path.find() != -1 or anEvent.src_path.find('/properties/') != -1):
+			if (anEvent.src_path.find(item) != -1 for item in ['html', 'properties']):
+				print "Updating resource files ..."
+				os.system(projectBaseDir() + '/scripts/builder/main.py --frontends beta gamma delta --backends dev')
+				subprocess.call('say done', shell=True)
+				print "DONE!"
+
 
 def main ():
-	site = ClipperzTestSite(proxy.ReverseProxyResource('localhost', 8080, '/java-backend'))
+#	site = ClipperzTestSite(proxy.ReverseProxyResource('localhost', 8080, '/java-backend'))
+	site = ClipperzTestSite(proxy.ReverseProxyResource('localhost', 8080, '/app'))
 #	site = ClipperzTestSite(proxy.ReverseProxyResource('www.clipperz.com', 443, '/'))
 	reactor.listenTCP(8888, site)
+
+#	event_handler = BuildScriptRunner()
+#	observer = Observer()
+#	observer.schedule(event_handler, path=projectBaseDir() + '/frontend', recursive=True)
+#	observer.start()
+
 	reactor.run()
 
 
 if __name__ == "__main__":
 	main()
-