terrain.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. # Copyright (C) 2011 Internet Systems Consortium.
  2. #
  3. # Permission to use, copy, modify, and distribute this software for any
  4. # purpose with or without fee is hereby granted, provided that the above
  5. # copyright notice and this permission notice appear in all copies.
  6. #
  7. # THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
  8. # DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
  9. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
  10. # INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
  11. # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
  12. # FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
  13. # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  14. # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15. #
  16. # This is the 'terrain' in which the lettuce lives. By convention, this is
  17. # where global setup and teardown is defined.
  18. #
  19. # We declare some attributes of the global 'world' variables here, so the
  20. # tests can safely assume they are present.
  21. #
  22. # We also use it to provide scenario invariants, such as resetting data.
  23. #
  24. from lettuce import *
  25. import subprocess
  26. import os
  27. import shutil
  28. import re
  29. import sys
  30. import time
  31. # lettuce cannot directly pass commands to the terrain, so we need to
  32. # use environment variables to influence behaviour
  33. KEEP_OUTPUT = 'LETTUCE_KEEP_OUTPUT'
  34. # In order to make sure we start all tests with a 'clean' environment,
  35. # We perform a number of initialization steps, like restoring configuration
  36. # files, and removing generated data files.
  37. # This approach may not scale; if so we should probably provide specific
  38. # initialization steps for scenarios. But until that is shown to be a problem,
  39. # It will keep the scenarios cleaner.
  40. # This is a list of files that are freshly copied before each scenario
  41. # The first element is the original, the second is the target that will be
  42. # used by the tests that need them
  43. copylist = [
  44. ["configurations/bindctl_commands.config.orig",
  45. "configurations/bindctl_commands.config"],
  46. ["configurations/example.org.config.orig",
  47. "configurations/example.org.config"],
  48. ["configurations/generate.config.orig",
  49. "configurations/generate.config"],
  50. ["configurations/bindctl/bindctl.config.orig",
  51. "configurations/bindctl/bindctl.config"],
  52. ["configurations/auth/auth_basic.config.orig",
  53. "configurations/auth/auth_basic.config"],
  54. ["configurations/auth/auth_badzone.config.orig",
  55. "configurations/auth/auth_badzone.config"],
  56. ["configurations/resolver/resolver_basic.config.orig",
  57. "configurations/resolver/resolver_basic.config"],
  58. ["configurations/multi_instance/multi_auth.config.orig",
  59. "configurations/multi_instance/multi_auth.config"],
  60. ["configurations/ddns/ddns.config.orig",
  61. "configurations/ddns/ddns.config"],
  62. ["configurations/ddns/noddns.config.orig",
  63. "configurations/ddns/noddns.config"],
  64. ["configurations/xfrin/retransfer_master.conf.orig",
  65. "configurations/xfrin/retransfer_master.conf"],
  66. ["configurations/xfrin/retransfer_master_v4.conf.orig",
  67. "configurations/xfrin/retransfer_master_v4.conf"],
  68. ["configurations/xfrin/retransfer_master_nons.conf.orig",
  69. "configurations/xfrin/retransfer_master_nons.conf"],
  70. ["configurations/xfrin/retransfer_slave.conf.orig",
  71. "configurations/xfrin/retransfer_slave.conf"],
  72. ["configurations/xfrin/retransfer_slave_notify.conf.orig",
  73. "configurations/xfrin/retransfer_slave_notify.conf"],
  74. ["configurations/root.config.orig",
  75. "configurations/root.config"],
  76. ["data/inmem-xfrin.sqlite3.orig",
  77. "data/inmem-xfrin.sqlite3"],
  78. ["data/root.sqlite3.orig",
  79. "data/root.sqlite3"],
  80. ["data/xfrin-before-diffs.sqlite3.orig",
  81. "data/xfrin-before-diffs.sqlite3"],
  82. ["data/xfrin-notify.sqlite3.orig",
  83. "data/xfrin-notify.sqlite3"],
  84. ["data/ddns/example.org.sqlite3.orig",
  85. "data/ddns/example.org.sqlite3"],
  86. ["data/empty_db.sqlite3",
  87. "data/xfrout.sqlite3"]
  88. ]
  89. # This is a list of files that, if present, will be removed before a scenario
  90. removelist = [
  91. "data/test_nonexistent_db.sqlite3"
  92. ]
  93. # When waiting for output data of a running process, use OUTPUT_WAIT_INTERVAL
  94. # as the interval in which to check again if it has not been found yet.
  95. # If we have waited OUTPUT_WAIT_MAX_INTERVALS times, we will abort with an
  96. # error (so as not to hang indefinitely)
  97. OUTPUT_WAIT_INTERVAL = 0.5
  98. OUTPUT_WAIT_MAX_INTERVALS = 120
  99. # class that keeps track of one running process and the files
  100. # we created for it.
  101. class RunningProcess:
  102. def __init__(self, step, process_name, args):
  103. # set it to none first so destructor won't error if initializer did
  104. """
  105. Initialize the long-running process structure, and start the process.
  106. Parameters:
  107. step: The scenario step it was called from. This is used for
  108. determining the output files for redirection of stdout
  109. and stderr.
  110. process_name: The name to refer to this running process later.
  111. args: Array of arguments to pass to Popen().
  112. """
  113. self.process = None
  114. self.step = step
  115. self.process_name = process_name
  116. self.remove_files_on_exit = (os.environ.get(KEEP_OUTPUT) != '1')
  117. self._check_output_dir()
  118. self._create_filenames()
  119. self._start_process(args)
  120. # used in _wait_for_output_str, map from (filename, (strings))
  121. # to a file offset.
  122. self.__file_offsets = {}
  123. def _start_process(self, args):
  124. """
  125. Start the process.
  126. Parameters:
  127. args:
  128. Array of arguments to pass to Popen().
  129. """
  130. stderr_write = open(self.stderr_filename, "w")
  131. stdout_write = open(self.stdout_filename, "w")
  132. self.process = subprocess.Popen(args, 0, None, subprocess.PIPE,
  133. stdout_write, stderr_write)
  134. # open them again, this time for reading
  135. self.stderr = open(self.stderr_filename, "r")
  136. self.stdout = open(self.stdout_filename, "r")
  137. def mangle_filename(self, filebase, extension):
  138. """
  139. Remove whitespace and non-default characters from a base string,
  140. and return the substituted value. Whitespace is replaced by an
  141. underscore. Any other character that is not an ASCII letter, a
  142. number, a dot, or a hyphen or underscore is removed.
  143. Parameter:
  144. filebase: The string to perform the substitution and removal on
  145. extension: An extension to append to the result value
  146. Returns the modified filebase with the given extension
  147. """
  148. filebase = re.sub("\s+", "_", filebase)
  149. filebase = re.sub("[^a-zA-Z0-9.\-_]", "", filebase)
  150. return filebase + "." + extension
  151. def _check_output_dir(self):
  152. # We may want to make this overridable by the user, perhaps
  153. # through an environment variable. Since we currently expect
  154. # lettuce to be run from our lettuce dir, we shall just use
  155. # the relative path 'output/'
  156. """
  157. Make sure the output directory for stdout/stderr redirection
  158. exists.
  159. Fails if it exists but is not a directory, or if it does not
  160. and we are unable to create it.
  161. """
  162. self._output_dir = os.getcwd() + os.sep + "output"
  163. if not os.path.exists(self._output_dir):
  164. os.mkdir(self._output_dir)
  165. assert os.path.isdir(self._output_dir),\
  166. self._output_dir + " is not a directory."
  167. def _create_filenames(self):
  168. """
  169. Derive the filenames for stdout/stderr redirection from the
  170. feature, scenario, and process name. The base will be
  171. "<Feature>-<Scenario>-<process name>.[stdout|stderr]"
  172. """
  173. filebase = self.step.scenario.feature.name + "-" +\
  174. self.step.scenario.name + "-" + self.process_name
  175. self.stderr_filename = self._output_dir + os.sep +\
  176. self.mangle_filename(filebase, "stderr")
  177. self.stdout_filename = self._output_dir + os.sep +\
  178. self.mangle_filename(filebase, "stdout")
  179. def stop_process(self):
  180. """
  181. Stop this process by calling terminate(). Blocks until process has
  182. exited. If remove_files_on_exit is True, redirected output files
  183. are removed.
  184. """
  185. if self.process is not None:
  186. self.process.terminate()
  187. self.process.wait()
  188. self.process = None
  189. if self.remove_files_on_exit:
  190. self._remove_files()
  191. def _remove_files(self):
  192. """
  193. Remove the files created for redirection of stdout/stderr output.
  194. """
  195. os.remove(self.stderr_filename)
  196. os.remove(self.stdout_filename)
  197. def _wait_for_output_str(self, filename, running_file, strings, only_new,
  198. matches=1):
  199. """
  200. Wait for a line of output in this process. This will (if
  201. only_new is False) check all output from the process including
  202. that may have been checked before. If only_new is True, it
  203. only checks output that has not been covered in previous calls
  204. to this method for the file (if there was no such previous call to
  205. this method, it works same as the case of only_new=False).
  206. Care should be taken if only_new is to be set to True, as it may cause
  207. counter-intuitive results. For example, assume the file is expected
  208. to contain a line that has XXX and another line has YYY, but the
  209. ordering is not predictable. If this method is called with XXX as
  210. the search string, but the line containing YYY appears before the
  211. target line, this method remembers the point in the file beyond
  212. the line that has XXX. If a next call to this method specifies
  213. YYY as the search string with only_new being True, the search will
  214. fail. If the same string is expected to appear multiple times
  215. and you want to catch the latest one, a more reliable way is to
  216. specify the match number and set only_new to False, if the number
  217. of matches is predictable.
  218. For each line in the output, the given strings array is checked. If
  219. any output lines checked contains one of the strings in the strings
  220. array, that string (not the line!) is returned.
  221. Parameters:
  222. filename: The filename to read previous output from, if applicable.
  223. running_file: The open file to read new output from.
  224. strings: Array of strings to look for.
  225. only_new: See above.
  226. matches: Check for the string this many times.
  227. Returns a tuple containing the matched string, and the complete line
  228. it was found in.
  229. Fails if none of the strings was read after 10 seconds
  230. (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
  231. """
  232. # Identify the start offset of search. if only_new=True, start from
  233. # the farthest point we've reached in the file; otherwise start from
  234. # the beginning.
  235. if not filename in self.__file_offsets:
  236. self.__file_offsets[filename] = 0
  237. offset = self.__file_offsets[filename] if only_new else 0
  238. running_file.seek(offset)
  239. match_count = 0
  240. wait_count = 0
  241. while wait_count < OUTPUT_WAIT_MAX_INTERVALS:
  242. line = running_file.readline()
  243. where = running_file.tell()
  244. if line:
  245. for string in strings:
  246. if line.find(string) != -1:
  247. match_count += 1
  248. if match_count >= matches:
  249. # If we've gone further, update the recorded offset
  250. if where > self.__file_offsets[filename]:
  251. self.__file_offsets[filename] = where
  252. return (string, line)
  253. else:
  254. wait_count += 1
  255. time.sleep(OUTPUT_WAIT_INTERVAL)
  256. running_file.seek(where)
  257. assert False, "Timeout waiting for process output: " + str(strings)
  258. def wait_for_stderr_str(self, strings, only_new = True, matches = 1):
  259. """
  260. Wait for one of the given strings in this process's stderr output.
  261. Parameters:
  262. strings: Array of strings to look for.
  263. only_new: See _wait_for_output_str.
  264. matches: Check for the string this many times.
  265. Returns a tuple containing the matched string, and the complete line
  266. it was found in.
  267. Fails if none of the strings was read after 10 seconds
  268. (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
  269. """
  270. return self._wait_for_output_str(self.stderr_filename, self.stderr,
  271. strings, only_new, matches)
  272. def wait_for_stdout_str(self, strings, only_new = True, matches = 1):
  273. """
  274. Wait for one of the given strings in this process's stdout output.
  275. Parameters:
  276. strings: Array of strings to look for.
  277. only_new: See _wait_for_output_str.
  278. matches: Check for the string this many times.
  279. Returns a tuple containing the matched string, and the complete line
  280. it was found in.
  281. Fails if none of the strings was read after 10 seconds
  282. (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
  283. """
  284. return self._wait_for_output_str(self.stdout_filename, self.stdout,
  285. strings, only_new, matches)
  286. # Container class for a number of running processes
  287. # i.e. servers like bind10, etc
  288. # one-shot programs like dig or bindctl are started and closed separately
  289. class RunningProcesses:
  290. def __init__(self):
  291. """
  292. Initialize with no running processes.
  293. """
  294. self.processes = {}
  295. def add_process(self, step, process_name, args):
  296. """
  297. Start a process with the given arguments, and store it under the given
  298. name.
  299. Parameters:
  300. step: The scenario step it was called from. This is used for
  301. determining the output files for redirection of stdout
  302. and stderr.
  303. process_name: The name to refer to this running process later.
  304. args: Array of arguments to pass to Popen().
  305. Fails if a process with the given name is already running.
  306. """
  307. assert process_name not in self.processes,\
  308. "Process " + process_name + " already running"
  309. self.processes[process_name] = RunningProcess(step, process_name, args)
  310. def get_process(self, process_name):
  311. """
  312. Return the Process with the given process name.
  313. Parameters:
  314. process_name: The name of the process to return.
  315. Fails if the process is not running.
  316. """
  317. assert process_name in self.processes,\
  318. "Process " + name + " unknown"
  319. return self.processes[process_name]
  320. def stop_process(self, process_name):
  321. """
  322. Stop the Process with the given process name.
  323. Parameters:
  324. process_name: The name of the process to return.
  325. Fails if the process is not running.
  326. """
  327. assert process_name in self.processes,\
  328. "Process " + name + " unknown"
  329. self.processes[process_name].stop_process()
  330. del self.processes[process_name]
  331. def stop_all_processes(self):
  332. """
  333. Stop all running processes.
  334. """
  335. for process in self.processes.values():
  336. process.stop_process()
  337. def keep_files(self):
  338. """
  339. Keep the redirection files for stdout/stderr output of all processes
  340. instead of removing them when they are stopped later.
  341. """
  342. for process in self.processes.values():
  343. process.remove_files_on_exit = False
  344. def wait_for_stderr_str(self, process_name, strings, only_new = True, matches = 1):
  345. """
  346. Wait for one of the given strings in the given process's stderr output.
  347. Parameters:
  348. process_name: The name of the process to check the stderr output of.
  349. strings: Array of strings to look for.
  350. only_new: See _wait_for_output_str.
  351. matches: Check for the string this many times.
  352. Returns the matched string.
  353. Fails if none of the strings was read after 10 seconds
  354. (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
  355. Fails if the process is unknown.
  356. """
  357. assert process_name in self.processes,\
  358. "Process " + process_name + " unknown"
  359. return self.processes[process_name].wait_for_stderr_str(strings,
  360. only_new,
  361. matches)
  362. def wait_for_stdout_str(self, process_name, strings, only_new = True, matches = 1):
  363. """
  364. Wait for one of the given strings in the given process's stdout output.
  365. Parameters:
  366. process_name: The name of the process to check the stdout output of.
  367. strings: Array of strings to look for.
  368. only_new: See _wait_for_output_str.
  369. matches: Check for the string this many times.
  370. Returns the matched string.
  371. Fails if none of the strings was read after 10 seconds
  372. (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
  373. Fails if the process is unknown.
  374. """
  375. assert process_name in self.processes,\
  376. "Process " + process_name + " unknown"
  377. return self.processes[process_name].wait_for_stdout_str(strings,
  378. only_new,
  379. matches)
  380. @before.each_scenario
  381. def initialize(scenario):
  382. """
  383. Global initialization for each scenario.
  384. """
  385. # Keep track of running processes
  386. world.processes = RunningProcesses()
  387. # Convenience variable to access the last query result from querying.py
  388. world.last_query_result = None
  389. # Convenience variable to access the last HTTP response from http.py
  390. world.last_http_response = None
  391. # For slightly better errors, initialize a process_pids for the relevant
  392. # steps
  393. world.process_pids = None
  394. # Some tests can modify the settings. If the tests fail half-way, or
  395. # don't clean up, this can leave configurations or data in a bad state,
  396. # so we copy them from originals before each scenario
  397. for item in copylist:
  398. shutil.copy(item[0], item[1])
  399. for item in removelist:
  400. if os.path.exists(item):
  401. os.remove(item)
  402. @after.each_scenario
  403. def cleanup(scenario):
  404. """
  405. Global cleanup for each scenario.
  406. """
  407. # Keep output files if the scenario failed
  408. if not scenario.passed:
  409. world.processes.keep_files()
  410. # Stop any running processes we may have had around
  411. world.processes.stop_all_processes()
  412. # Environment check
  413. # Checks if LETTUCE_SETUP_COMPLETED is set in the environment
  414. # If not, abort with an error to use the run-script
  415. if 'LETTUCE_SETUP_COMPLETED' not in os.environ:
  416. print("Environment check failure; LETTUCE_SETUP_COMPLETED not set")
  417. print("Please use the run_lettuce.sh script. If you want to test an")
  418. print("installed version of bind10 with these tests, use")
  419. print("run_lettuce.sh -I [lettuce arguments]")
  420. sys.exit(1)