#!/usr/bin/env python
#
# wmllint -- check WML for conformance to the most recent dialect
#
# By Eric S. Raymond April 2007.
#
# All conversion logic for lifting WML and maps from older versions of the
# markup to newer ones should live here.  This includes resource path changes
# and renames, also map format conversions.
#
# Note: Lift logic for pre-1.4 versions has been removed; if you need
# it, check out a copy of wmllint from the 1.4 stable branch and use
# that to lift before running this one.  I did this for a policy
# reason; I wanted to kill off the --oldversion switch.  It will *not*
# be restored; in future, changes to WML syntax *must* be forward
# compatible in such a way that tags from old versions can be
# unambiguously recognized (this will save everybody heartburn).  As a
# virtuous side effect, this featurectomy cuts wmllint's code
# complexity by over 50%, improves performance by about 33%, and
# banishes some annoying behaviors related to the 1.2 map-conversion
# code.
#
# While the script is at it, it checks for various incorrect and dodgy WML
# constructs, including:
#   * unbalanced tags
#   * strings that need a translation mark and should not have them
#   * strings that have a translation mark and should not
#   * translatable strings containing macro references 
#   * filter references by id= not matched by an actual unit
#   * abilities or traits without matching special notes, or vice-versa
#   * consistency between recruit= and recruitment_pattern= instances
#   * unknown unit types in recruitment lists
#   * double space after punctuation in translatable strings.
#   * unknown races or movement types in units
#   * unknown base units
#   * misspellings in message and description strings
#
# Takes any number of directories as arguments.  Each directory is converted.
# If no directories are specified, acts on the current directory.
#
# The recommended procedure is this:
# 1. Run it with --dryrun first to see what it will do.
# 2. If the messages look good, run without --dryrun; the old content
#    will be left in backup files with a -bak extension.
# 3. Eyeball the changes with the --diff option.
# 4. Use wmlscope, with a directory list including the Wesnoth mainline WML
#    as first argument, to check that you have no unresolved references.
# 5. Test the conversion.
# 6. Use either --clean to remove the -bak files or --revert to
#    undo the conversion.
#
# Standalone terrain mask files *must* have a .mask extension on their name
# or they'll have an incorrect usage=map generated into them.
#
# Note: You can shut wmllint up about custom terrains by having a comment
# on the same line that includes the string "wmllint: ignore" or
# "wmllint: noconvert". The same magic comments will also disable checking
# of translation marks.
#
# You can also prevent description insertions with "wmllint: no-icon".
#
# You can force otherwise undeclared characters to be recogized with
# a magic comment containing the string "wmllint: recognize".
# The rest of the line is stripped and treated as the name of a character
# who should be recognized in descriptions.  This will be useful,
# for example, if your scenario follows a continue so there are
# characters present who were not explicitly recalled.  It may
# also be useful if you have wrapped unit-creation or recall markup in macros
# and wmllint cannot recognize it.
#
# Similarly, it is possible to explicitly declare a unit's usage class
# with a magic comment that looks like this:
#    wmllint: usage of <unit> is <class>
# Note that <unit> must be a string wrapped in ASCII doublequotes.  This
# declaration will be useful if you are declaring units with macros that
# include a substitutable formal in the unit name; there are examples in UtBS.
#
# You can disable stack-based malformation checks with a comment
# containing "wmllint: validate-off" and re-enable with "wmllint: validate-on".
#
# You can prevent filename conversions with a comment containing
# "wmllint: noconvert" on the same line as the filename.
#
# You can suppress complaints about files without an initial textdoman line
# by embedding the magic comment "# wmllint: no translatables" in the file.
# of course, it's a good idea to be sure this assertion is actually true.
#
# You can skip checks on unbalanced WML (e.g. in a macro definition) by
# bracketing it with "wmllint: unbalanced-on" and "wmllint: unbalanced-off".
# Note that this will also disable stack-based validation on the span
# of lines they enclose.
#
# You can suppress warnings about newlines in messages (and attempts to
# repair them) with "wmllint: display on", and re-enable them with
# "wmllint: display off".  The repair attempts (only) may also be
# suppressed with the --stringfreeze option.
#
# A special comment "#wmllint: notecheck off" will disable error messages on
# traits with no corresponding {SPECIAL_NOTE} explanations.  The comment
# "#wmllint: notecheck on" will re-enable this check.
#
# A magic comment of the form "wmllint: general spellings word1
# word2..."  will declare the tokens word1, word2, etc. to be
# acceptable spellings for anywhere in the Wesnoth tree that the
# spellchecker should rever flag.  If the keyword "general" is
# replaced by "local", the spelling exceptions apply only in the
# current file. If the keyword "general" is replaced by "directory",
# the spelling exceptions apply to all files below the parent
# directory.
#
# A comment containing "no spellcheck" disables spellchecking on the
# line where it occurs.
#
# A comment of the form 
#
# #wmllint: match {ABILITY_FOO} with {SPECIAL_NOTES_IOO}
#
# will declare an ability macro and a special-notes macro to be tied
# together for reference-checking purposes.

import sys, os, re, getopt, string, copy, difflib, time, gzip
from wesnoth.wmltools import *
from wesnoth.wmliterator import *

# Global changes meant to be done on all lines.  Suppressed by noconvert.
linechanges = (
        ("canrecruit=1", "canrecruit=yes"),
        ("canrecruit=0", "canrecruit=no"),
        ("generate_description", "generate_name"),
        # Fix a common typo
        ("agression=", "aggression="),
        # These changed just after 1.5.0
        ("[special_filter]", "[filter_attack]"),
        ("[wml_filter]", "[filter_wml]"),
        ("[unit_filter]", "[filter]"),
        ("[secondary_unit_filter]", "[filter_second]"),
        ("[attack_filter]", "[filter_attack]"),
        ("[secondary_attack_filter]", "[filter_second_attack]"),
        ("[special_filter_second]", "[filter_second_attack]"),
        ("[/special_filter]", "[/filter_attack]"),
        ("[/wml_filter]", "[/filter_wml]"),
        ("[/unit_filter]", "[/filter]"),
        ("[/secondary_unit_filter]", "[/filter_second]"),
        ("[/attack_filter]", "[/filter_attack]"),
        ("[/secondary_attack_filter]", "[/filter_second_attack]"),
        ("[/special_filter_second]", "[/filter_second_attack]"),
        ("grassland=", "flat="),
        ("tundra=", "frozen="),
        ("cavewall=", "impassable="),
        ("canyon=", "unwalkable="),
        # This changed after 1.5.2
        ("advanceto=", "advances_to="),
        # This changed after 1.5.5, to enable mechanical spellchecking
        ("sabre", "saber"),
        ("nr-sad.ogg", "sad.ogg"),
        # Changed after 1.5.7
        ("[debug_message]", "[wml_message]"),
        ("[/debug_message]", "[/wml_message]"),
        # Changed just before 1.5.9
        ("portraits/Alex_Jarocha-Ernst/drake-burner.png",
         "portraits/drakes/burner.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-clasher.png",
         "portraits/drakes/clasher.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-fighter.png",
         "portraits/drakes/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-glider.png",
         "portraits/drakes/glider.png"),
        ("portraits/Alex_Jarocha-Ernst/ghoul.png",
         "portraits/undead/ghoul.png"),
        ("portraits/Alex_Jarocha-Ernst/mermaid-initiate.png",
         "portraits/merfolk/initiate.png"),
        ("portraits/Alex_Jarocha-Ernst/merman-fighter.png",
         "portraits/merfolk/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/merman-hunter.png",
         "portraits/merfolk/hunter.png"),
        ("portraits/Alex_Jarocha-Ernst/naga-fighter.png",
         "portraits/nagas/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/nagini-fighter.png",
         "portraits/nagas/fighter+female.png"),
        ("portraits/Alex_Jarocha-Ernst/orcish-assassin.png",
         "portraits/orcs/assassin.png"),
        ("portraits/Emilien_Rotival/human-general.png",
         "portraits/humans/general.png"),
        ("portraits/Emilien_Rotival/human-heavyinfantry.png",
         "portraits/humans/heavy-infantry.png"),
        ("portraits/Emilien_Rotival/human-ironmauler.png",
         "portraits/humans/iron-mauler.png"),
        ("portraits/Emilien_Rotival/human-lieutenant.png",
         "portraits/humans/lieutenant.png"),
        ("portraits/Emilien_Rotival/human-marshal.png",
         "portraits/humans/marshal.png"),
        ("portraits/Emilien_Rotival/human-peasant.png",
         "portraits/humans/peasant.png"),
        ("portraits/Emilien_Rotival/human-pikeman.png",
         "portraits/humans/pikeman.png"),
        ("portraits/Emilien_Rotival/human-royalguard.png",
         "portraits/humans/royal-guard.png"),
        ("portraits/Emilien_Rotival/human-sergeant.png",
         "portraits/humans/sergeant.png"),
        ("portraits/Emilien_Rotival/human-spearman.png",
         "portraits/humans/spearman.png"),
        ("portraits/Emilien_Rotival/human-swordsman.png",
         "portraits/humans/swordsman.png"),
        ("portraits/Emilien_Rotival/transparent/human-general.png",
         "portraits/humans/transparent/general.png"),
        ("portraits/Emilien_Rotival/transparent/human-heavyinfantry.png",
         "portraits/humans/transparent/heavy-infantry.png"),
        ("portraits/Emilien_Rotival/transparent/human-ironmauler.png",
         "portraits/humans/transparent/iron-mauler.png"),
        ("portraits/Emilien_Rotival/transparent/human-lieutenant.png",
         "portraits/humans/transparent/lieutenant.png"),
        ("portraits/Emilien_Rotival/transparent/human-marshal.png",
         "portraits/humans/transparent/marshal.png"),
        ("portraits/Emilien_Rotival/transparent/human-marshal-2.png",
         "portraits/humans/transparent/marshal-2.png"),
        ("portraits/Emilien_Rotival/transparent/human-peasant.png",
         "portraits/humans/transparent/peasant.png"),
        ("portraits/Emilien_Rotival/transparent/human-pikeman.png",
         "portraits/humans/transparent/pikeman.png"),
        ("portraits/Emilien_Rotival/transparent/human-royalguard.png",
         "portraits/humans/transparent/royal-guard.png"),
        ("portraits/Emilien_Rotival/transparent/human-sergeant.png",
         "portraits/humans/transparent/sergeant.png"),
        ("portraits/Emilien_Rotival/transparent/human-spearman.png",
         "portraits/humans/transparent/spearman.png"),
        ("portraits/Emilien_Rotival/transparent/human-swordsman.png",
         "portraits/humans/transparent/swordsman.png"),
        ("portraits/James_Woo/assassin.png",
         "portraits/humans/assassin.png"),
        ("portraits/James_Woo/dwarf-guard.png",
         "portraits/dwarves/guard.png"),
        ("portraits/James_Woo/orc-warlord.png",
         "portraits/orcs/warlord.png"),
        ("portraits/James_Woo/orc-warlord2.png",
         "portraits/orcs/warlord2.png"),
        ("portraits/James_Woo/orc-warlord3.png",
         "portraits/orcs/warlord3.png"),
        ("portraits/James_Woo/orc-warlord4.png",
         "portraits/orcs/warlord4.png"),
        ("portraits/James_Woo/orc-warlord5.png",
         "portraits/orcs/warlord5.png"),
        ("portraits/James_Woo/troll.png",
         "portraits/trolls/troll.png"),
        ("portraits/Jason_Lutes/human-bandit.png",
         "portraits/humans/bandit.png"),
        ("portraits/Jason_Lutes/human-grand-knight.png",
         "portraits/humans/grand-knight.png"),
        ("portraits/Jason_Lutes/human-halberdier.png",
         "portraits/humans/halberdier.png"),
        ("portraits/Jason_Lutes/human-highwayman.png",
         "portraits/humans/highwayman.png"),
        ("portraits/Jason_Lutes/human-horseman.png",
         "portraits/humans/horseman.png"),
        ("portraits/Jason_Lutes/human-javelineer.png",
         "portraits/humans/javelineer.png"),
        ("portraits/Jason_Lutes/human-knight.png",
         "portraits/humans/knight.png"),
        ("portraits/Jason_Lutes/human-lancer.png",
         "portraits/humans/lancer.png"),
        ("portraits/Jason_Lutes/human-paladin.png",
         "portraits/humans/paladin.png"),
        ("portraits/Jason_Lutes/human-thug.png",
         "portraits/humans/thug.png"),
        ("portraits/Kitty/elvish-archer.png",
         "portraits/elves/archer.png"),
        ("portraits/Kitty/elvish-archer+female.png",
         "portraits/elves/archer+female.png"),
        ("portraits/Kitty/elvish-captain.png",
         "portraits/elves/captain.png"),
        ("portraits/Kitty/elvish-druid.png",
         "portraits/elves/druid.png"),
        ("portraits/Kitty/elvish-fighter.png",
         "portraits/elves/fighter.png"),
        ("portraits/Kitty/elvish-hero.png",
         "portraits/elves/hero.png"),
        ("portraits/Kitty/elvish-high-lord.png",
         "portraits/elves/high-lord.png"),
        ("portraits/Kitty/elvish-lady.png",
         "portraits/elves/lady.png"),
        ("portraits/Kitty/elvish-lord.png",
         "portraits/elves/lord.png"),
        ("portraits/Kitty/elvish-marksman.png",
         "portraits/elves/marksman.png"),
        ("portraits/Kitty/elvish-marksman+female.png",
         "portraits/elves/marksman+female.png"),
        ("portraits/Kitty/elvish-ranger.png",
         "portraits/elves/ranger.png"),
        ("portraits/Kitty/elvish-ranger+female.png",
         "portraits/elves/ranger+female.png"),
        ("portraits/Kitty/elvish-scout.png",
         "portraits/elves/scout.png"),
        ("portraits/Kitty/elvish-shaman.png",
         "portraits/elves/shaman.png"),
        ("portraits/Kitty/elvish-shyde.png",
         "portraits/elves/shyde.png"),
        ("portraits/Kitty/elvish-sorceress.png",
         "portraits/elves/sorceress.png"),
        ("portraits/Kitty/human-dark-adept.png",
         "portraits/humans/dark-adept.png"),
        ("portraits/Kitty/human-dark-adept+female.png",
         "portraits/humans/dark-adept+female.png"),
        ("portraits/Kitty/human-mage.png",
         "portraits/humans/mage.png"),
        ("portraits/Kitty/human-mage+female.png",
         "portraits/humans/mage+female.png"),
        ("portraits/Kitty/human-mage-arch.png",
         "portraits/humans/mage-arch.png"),
        ("portraits/Kitty/human-mage-arch+female.png",
         "portraits/humans/mage-arch+female.png"),
        ("portraits/Kitty/human-mage-light.png",
         "portraits/humans/mage-light.png"),
        ("portraits/Kitty/human-mage-light+female.png",
         "portraits/humans/mage-light+female.png"),
        ("portraits/Kitty/human-mage-red.png",
         "portraits/humans/mage-red.png"),
        ("portraits/Kitty/human-mage-red+female.png",
         "portraits/humans/mage-red+female.png"),
        ("portraits/Kitty/human-mage-silver.png",
         "portraits/humans/mage-silver.png"),
        ("portraits/Kitty/human-mage-silver+female.png",
         "portraits/humans/mage-silver+female.png"),
        ("portraits/Kitty/human-mage-white.png",
         "portraits/humans/mage-white.png"),
        ("portraits/Kitty/human-mage-white+female.png",
         "portraits/humans/mage-white+female.png"),
        ("portraits/Kitty/human-necromancer.png",
         "portraits/humans/necromancer.png"),
        ("portraits/Kitty/human-necromancer+female.png",
         "portraits/humans/necromancer+female.png"),
        ("portraits/Kitty/troll-whelp.png",
         "portraits/trolls/whelp.png"),
        ("portraits/Kitty/undead-lich.png",
         "portraits/undead/lich.png"),
        ("portraits/Kitty/transparent/elvish-archer.png",
         "portraits/elves/transparent/archer.png"),
        ("portraits/Kitty/transparent/elvish-archer+female.png",
         "portraits/elves/transparent/archer+female.png"),
        ("portraits/Kitty/transparent/elvish-captain.png",
         "portraits/elves/transparent/captain.png"),
        ("portraits/Kitty/transparent/elvish-druid.png",
         "portraits/elves/transparent/druid.png"),
        ("portraits/Kitty/transparent/elvish-fighter.png",
         "portraits/elves/transparent/fighter.png"),
        ("portraits/Kitty/transparent/elvish-hero.png",
         "portraits/elves/transparent/hero.png"),
        ("portraits/Kitty/transparent/elvish-high-lord.png",
         "portraits/elves/transparent/high-lord.png"),
        ("portraits/Kitty/transparent/elvish-lady.png",
         "portraits/elves/transparent/lady.png"),
        ("portraits/Kitty/transparent/elvish-lord.png",
         "portraits/elves/transparent/lord.png"),
        ("portraits/Kitty/transparent/elvish-marksman.png",
         "portraits/elves/transparent/marksman.png"),
        ("portraits/Kitty/transparent/elvish-marksman+female.png",
         "portraits/elves/transparent/marksman+female.png"),
        ("portraits/Kitty/transparent/elvish-ranger.png",
         "portraits/elves/transparent/ranger.png"),
        ("portraits/Kitty/transparent/elvish-ranger+female.png",
         "portraits/elves/transparent/ranger+female.png"),
        ("portraits/Kitty/transparent/elvish-scout.png",
         "portraits/elves/transparent/scout.png"),
        ("portraits/Kitty/transparent/elvish-shaman.png",
         "portraits/elves/transparent/shaman.png"),
        ("portraits/Kitty/transparent/elvish-shyde.png",
         "portraits/elves/transparent/shyde.png"),
        ("portraits/Kitty/transparent/elvish-sorceress.png",
         "portraits/elves/transparent/sorceress.png"),
        ("portraits/Kitty/transparent/human-dark-adept.png",
         "portraits/humans/transparent/dark-adept.png"),
        ("portraits/Kitty/transparent/human-dark-adept+female.png",
         "portraits/humans/transparent/dark-adept+female.png"),
        ("portraits/Kitty/transparent/human-mage.png",
         "portraits/humans/transparent/mage.png"),
        ("portraits/Kitty/transparent/human-mage+female.png",
         "portraits/humans/transparent/mage+female.png"),
        ("portraits/Kitty/transparent/human-mage-arch.png",
         "portraits/humans/transparent/mage-arch.png"),
        ("portraits/Kitty/transparent/human-mage-arch+female.png",
         "portraits/humans/transparent/mage-arch+female.png"),
        ("portraits/Kitty/transparent/human-mage-light.png",
         "portraits/humans/transparent/mage-light.png"),
        ("portraits/Kitty/transparent/human-mage-light+female.png",
         "portraits/humans/transparent/mage-light+female.png"),
        ("portraits/Kitty/transparent/human-mage-red.png",
         "portraits/humans/transparent/mage-red.png"),
        ("portraits/Kitty/transparent/human-mage-red+female.png",
         "portraits/humans/transparent/mage-red+female.png"),
        ("portraits/Kitty/transparent/human-mage-silver.png",
         "portraits/humans/transparent/mage-silver.png"),
        ("portraits/Kitty/transparent/human-mage-silver+female.png",
         "portraits/humans/transparent/mage-silver+female.png"),
        ("portraits/Kitty/transparent/human-mage-white.png",
         "portraits/humans/transparent/mage-white.png"),
        ("portraits/Kitty/transparent/human-mage-white+female.png",
         "portraits/humans/transparent/mage-white+female.png"),
        ("portraits/Kitty/transparent/human-necromancer.png",
         "portraits/humans/transparent/necromancer.png"),
        ("portraits/Kitty/transparent/human-necromancer+female.png",
         "portraits/humans/transparent/necromancer+female.png"),
        ("portraits/Kitty/transparent/troll-whelp.png",
         "portraits/trolls/transparent/whelp.png"),
        ("portraits/Kitty/transparent/undead-lich.png",
         "portraits/undead/transparent/lich.png"),
        ("portraits/Nicholas_Kerpan/human-poacher.png",
         "portraits/humans/poacher.png"),
        ("portraits/Nicholas_Kerpan/human-thief.png",
         "portraits/humans/thief.png"),
        ("portraits/Other/brown-lich.png",
         "portraits/undead/brown-lich.png"),
        ("portraits/Other/cavalryman.png",
         "portraits/humans/cavalryman.png"),
        ("portraits/Other/human-masterbowman.png",
         "portraits/humans/master-bowman.png"),
        ("portraits/Other/scorpion.png",
         "portraits/monsters/scorpion.png"),
        ("portraits/Other/sea-serpent.png",
         "portraits/monsters/sea-serpent.png"),
        ("portraits/Pekka_Aikio/human-bowman.png",
         "portraits/humans/bowman.png"),
        ("portraits/Pekka_Aikio/human-longbowman.png",
         "portraits/humans/longbowman.png"),
        ("portraits/Philip_Barber/dwarf-dragonguard.png",
         "portraits/dwarves/dragonguard.png"),
        ("portraits/Philip_Barber/dwarf-fighter.png",
         "portraits/dwarves/fighter.png"),
        ("portraits/Philip_Barber/dwarf-lord.png",
         "portraits/dwarves/lord.png"),
        ("portraits/Philip_Barber/dwarf-thunderer.png",
         "portraits/dwarves/thunderer.png"),
        ("portraits/Philip_Barber/saurian-augur.png",
         "portraits/saurians/augur.png"),
        ("portraits/Philip_Barber/saurian-skirmisher.png",
         "portraits/saurians/skirmisher.png"),
        ("portraits/Philip_Barber/undead-death-knight.png",
         "portraits/undead/death-knight.png"),
        ("portraits/Philip_Barber/transparent/dwarf-dragonguard.png",
         "portraits/dwarves/transparent/dragonguard.png"),
        ("portraits/Philip_Barber/transparent/dwarf-fighter.png",
         "portraits/dwarves/transparent/fighter.png"),
        ("portraits/Philip_Barber/transparent/dwarf-lord.png",
         "portraits/dwarves/transparent/lord.png"),
        ("portraits/Philip_Barber/transparent/dwarf-thunderer.png",
         "portraits/dwarves/transparent/thunderer.png"),
        ("portraits/Philip_Barber/transparent/saurian-augur.png",
         "portraits/saurians/transparent/augur.png"),
        ("portraits/Philip_Barber/transparent/saurian-skirmisher.png",
         "portraits/saurians/transparent/skirmisher.png"),
        ("portraits/Philip_Barber/transparent/undead-death-knight.png",
         "portraits/undead/transparent/death-knight.png"),
        # Changed just before 1.5.11
        ("titlescreen/landscapebattlefield.jpg",
         "story/landscape-battlefield.jpg"),
        ("titlescreen/landscapebridge.jpg",
         "story/landscape-bridge.jpg"),
        ("titlescreen/landscapecastle.jpg",
         "story/landscape-castle.jpg"),
        ("LABEL_PERSISTANT", "LABEL_PERSISTENT"),
        # Changed just before 1.5.13
        ("targetting", "targeting"),
        # Changed just after 1.7 fork
        ("[stone]", "[petrify]"),
        ("[unstone]", "[unpetrify]"),
        ("[/stone]", "[/petrify]"),
        ("[/unstone]", "[/unpetrify]"),
        ("WEAPON_SPECIAL_STONE", "WEAPON_SPECIAL_PETRIFY"),
        ("SPECIAL_NOTE_STONE", "SPECIAL_NOTE_PETRIFY"),
        (".stoned", ".petrified"),
        ("stoned=", "petrified="),
        # Changed at rev 37390
        ("swing=", "value_second="),
        # Changed just before 1.7.3
        ("Drake Gladiator", "Drake Thrasher"),
        ("gladiator-", "thrasher-"),
        ("Drake Slasher", "Drake Arbiter"),
        ("slasher-", "arbiter-"),
        # Changes after 1.7.5
        ("portraits/nagas/fighter+female.png", "portraits/nagas/fighter.png"),
        # Changes after 1.8rc1
        ("portraits/orcs/warlord.png", "portraits/orcs/transparent/warlord.png"),
        ("portraits/orcs/warlord2.png","portraits/orcs/transparent/warlord.png"),
        ("portraits/orcs/warlord3.png","portraits/orcs/transparent/grunt-2.png"),
        ("portraits/orcs/warlord4.png","portraits/orcs/transparent/grunt-2.png"),
        ("portraits/orcs/warlord5.png","portraits/orcs/transparent/grunt-3.png"),
        ("second_value=", "value_second="),    # Correct earlier wmllint error
        (".stones", ".petrifies"),
        ("stones=", "petrifies="),
        )

def validate_on_pop(tagstack, closer, filename, lineno):
    "Validate the stack at the time a new close tag is seen."
    (tag, attributes) = tagstack[-1]
    ancestors = map(lambda x: x[0], tagstack)
    if verbose >= 3:
        print '"%s", line %d: closing %s I see %s with %s' % (filename, lineno, closer, tag, attributes)
    # Detect a malformation that will cause the game to barf while attempting
    # to deserialize an empty unit.
    if "scenario" in ancestors and closer == "side" and "type" not in attributes and ("no_leader" not in attributes or attributes["no_leader"] != "yes") and "multiplayer" not in ancestors:
        print '"%s", line %d: [side] without type attribute' % (filename, lineno)
    # This assumes that conversion will always happen in units/ files.
    if "units" not in filename and closer == "unit" and "race" in attributes:
        print '"%s", line %d: [unit] needs hand fixup to [unit_type]' % \
              (filename, lineno)
    if closer == "campaign" and "id" not in attributes:
        print '"%s", line %d: campaign has no ID' % \
              (filename, lineno)
    if closer == "terrain" and attributes.get("heals") in ("true", "false"):
        print '"%s", line %d: heals attribute no longer takes a boolean' % \
              (filename, lineno)
    if closer == "unit" and attributes.get("id") is not None and attributes.get("type") is not None and attributes.get("side") is None:
        print '"%s", line %d: unit declaration without side attribute' % \
              (filename, lineno)

def within(tag):
    "Did the specified tag lead one of our enclosing contexts?"
    if type(tag) == type(()):	# Can take a list.
        for t in tag:
            if within(t):
                return True
        else:
            return False
    else:
        return tag in map(lambda x: x[0], tagstack)

def under(tag):
    "Did the specified tag lead the latest context?"
    if type(tag) == type(()):	# Can take a list.
        for t in tag:
            if within(t):
                return True
        else:
            return False
    elif tagstack:
        return tag == tagstack[-1][0]
    else:
        return False

def standard_unit_filter():
    "Are we within the syntactic context of a standard unit filter?"
    # It's under("message") rather than within("message") because
    # [message] can contain [option] markup with menu item description=
    # attributes that should not be altered.
    return within(("filter", "filter_second",
                   "filter_adjacent", "filter_opponent",
                   "unit_filter", "secondary_unit_filter",
                   "special_filter", "special_filter_second",
                   "neighbor_unit_filter",
                   "recall", "teleport", "kill", "unstone", "store_unit",
                   "have_unit", "scroll_to_unit", "role",
                   "hide_unit", "unhide_unit",
                   "protect_unit", "target", "avoid")) \
                   or under("message")

# Sanity checking

# Associations for the ability sanity checks.
# Note: Depends on ABILITY_EXTRA_HEAL not occurring outside ABILITY_CURES.
notepairs = [
    ("movement_type=undeadspirit", "{SPECIAL_NOTES_SPIRIT}"),
    ("type=arcane", "{SPECIAL_NOTES_ARCANE}"),
    ("{ABILITY_HEALS}", "{SPECIAL_NOTES_HEALS}"),
    #("{ABILITY_EXTRA_HEAL}", "{SPECIAL_NOTES_EXTRA_HEAL}"),
    ("{ABILITY_UNPOISON}", "{SPECIAL_NOTES_UNPOISON}"),
    ("{ABILITY_CURES}", "{SPECIAL_NOTES_CURES}"),
    ("{ABILITY_REGENERATES}", "{SPECIAL_NOTES_REGENERATES}"),
    ("{ABILITY_STEADFAST}", "{SPECIAL_NOTES_STEADFAST}"),
    ("{ABILITY_LEADERSHIP_LEVEL_", "{SPECIAL_NOTES_LEADERSHIP}"), # No } deliberately
    ("{ABILITY_SKIRMISHER}", "{SPECIAL_NOTES_SKIRMISHER}"),
    ("{ABILITY_ILLUMINATES}", "{SPECIAL_NOTES_ILLUMINATES}"),
    ("{ABILITY_TELEPORT}", "{SPECIAL_NOTES_TELEPORT}"),
    ("{ABILITY_AMBUSH}", "{SPECIAL_NOTES_AMBUSH}"),
    ("{ABILITY_NIGHTSTALK}", "{SPECIAL_NOTES_NIGHTSTALK}"),
    ("{ABILITY_CONCEALMENT}", "{SPECIAL_NOTES_CONCEALMENT}"),
    ("{ABILITY_SUBMERGE}", "{SPECIAL_NOTES_SUBMERGE}"),
    ("{ABILITY_FEEDING}", "{SPECIAL_NOTES_FEEDING}"),
    ("{WEAPON_SPECIAL_BERSERK}", "{SPECIAL_NOTES_BERSERK}"),
    ("{WEAPON_SPECIAL_BACKSTAB}", "{SPECIAL_NOTES_BACKSTAB}"),
    ("{WEAPON_SPECIAL_PLAGUE", "{SPECIAL_NOTES_PLAGUE}"),	# No } deliberately
    ("{WEAPON_SPECIAL_SLOW}", "{SPECIAL_NOTES_SLOW}"),
    ("{WEAPON_SPECIAL_PETRIFY}", "{SPECIAL_NOTES_PETRIFY}"),
    ("{WEAPON_SPECIAL_MARKSMAN}", "{SPECIAL_NOTES_MARKSMAN}"),
    ("{WEAPON_SPECIAL_MAGICAL}", "{SPECIAL_NOTES_MAGICAL}"),
    ("{WEAPON_SPECIAL_SWARM}", "{SPECIAL_NOTES_SWARM}"),
    ("{WEAPON_SPECIAL_CHARGE}", "{SPECIAL_NOTES_CHARGE}"),
    ("{WEAPON_SPECIAL_DRAIN}", "{SPECIAL_NOTES_DRAIN}"),
    ("{WEAPON_SPECIAL_FIRSTSTRIKE}", "{SPECIAL_NOTES_FIRSTSTRIKE}"),
    ("{WEAPON_SPECIAL_POISON}", "{SPECIAL_NOTES_POISON}"),
    ]


# This needs to match the list of usage types in ai_python.cpp
usage_types = ("scout", "fighter", "mixed fighter", "archer", "healer")

# These are accumulated by sanity_check() and examined by consistency_check()
unit_types = []
derived_units = []
usage = {}
sides = []
advances = []
movetypes = []
unit_movetypes = []
races = []
unit_races = []

# Attributes that should have translation marks
translatables = (\
               "abbrev",
               "cannot_use_message",
               "caption",
               "current_player",
               "description",
               "description_inactive",
               "end_text",
               "difficulty_descriptions",
               "female_name_inactive",
               "female_names",
               "label",
               "male_names",
               "message",
               "name",
               "name_inactive",
               "note",
               "order",
               "plural_name",
               "prefix",
               "set_description",
               "source",
               "story",
               "summary",
               "text", 
               "title",
               "title2",
               "tooltip",
               "translator_comment",
               "user_team_name",
               )

spellcheck_these = (\
    "cannot_use_message=",
    "caption=",
    "description=",
    "description_inactive=",
    "end_text=",
    "message=",
    "note=",
    "story=",
    "summary=",
    "text=",
    "title=",
    "title2=",
    "tooltip=",
    "user_team_name=",
    )

# Declare a few common English contractions and ejaculations that pyenchant
# inexplicably knows nothing of.
declared_spellings = {"GLOBAL":["I'm", "I've", "I'd", "I'll",
                                "heh", "ack",
                                # Fantasy/SF/occult jargon that we need
                                "aerie",
                                "aeon",
                                "aide-de-camp",
                                "axe",
                                "ballista",
                                "bided",
                                "crafters",
                                "glaive",
                                "greatsword",
                                "hellspawn",
                                "hurrah",
                                "morningstar",
                                "numbskulls",
                                "overmatched",
                                "stygian",
                                "teleport",
                                "teleportation",
                                "teleported",
                                "terraform",
                                "wildlands",
                                # game jargon
                                "melee", "arcane", "day/night", "gameplay",
                                "hitpoint", "hitpoints", "FFA", "multiplayer",
                                "playtesting", "respawn", "respawns",
                                "WML", "HP", "XP", "AI", "ZOC", "YW",
                                "L0", "L1", "L2", "L3",
                                # archaisms
                                "faugh", "hewn", "leapt", "dreamt", "spilt",
                                "grandmam", "grandsire", "grandsires",
                                "scry", "scrying", "scryed", "woodscraft",
                                "princeling", "wilderlands", "ensorcels",
                                # Sceptre of Fire gets spelled with -re.
                                "sceptre",
                                ]}

pango_conversions = (("~", "<b>", "</b>"),
                     ("@", "<span color='green'>", "</span>"),
                     ("#", "<span color='red'>", "</span>"),
                     ("*", "<span size='large'>", "</span>"),
                     ("`", "<span size='small'>", "</span>"),
                     )

def pangostrip(message):
    "Strip Pango margup out of a string."
    # This is all known Pango convenience tags
    for tag in ("b", "big", "i", "s", "sub", "sup", "small", "tt", "u"):
        message = message.replace("<"+tag+">", "").replace("</"+tag+">", "")
    # Now remove general span tags
    message = re.sub("</?span[^>]*>", "", message)
    # And Pango specials;
    message = re.sub("&[a-z]+;", "", message)
    return message

def pangoize(message, filename, line):
    "Pango conversion of old-style Wesnoth markup."
    if '&' in message:
        amper = message.find('&')
        if message[amper:amper+1].isspace():
            message = message[:amper] + "&amp;" + message[amper+1:]
    if re.search("<[0-9]+,[0-9]+,[0-9]+>", message):
        print '"%s", line %d: color spec in line requires manual fix.' % (filename, line)
    # Hack old-style Wesnoth markup
    for (oldstyle, newstart, newend) in pango_conversions:
        if oldstyle not in message:
            continue
        where = message.find(oldstyle)
        if message[where-1] != '"':	# Start of string only
            continue
        if message.strip()[-1] != '"':
            print '"%s", line %d: %s highlight at start of multiline string requires manual fix.' % (filename, line, oldstyle)
            continue
        if '+' in message:
            print '"%s", line %d: %s highlight in composite string requires manual fix.' % (filename, line, oldstyle)
            continue
        # This is the common, simple case we can fix automatically
        message = message[:where] + newstart + message[where+1:]
        endq = message.rfind('"')
        message = message[:endq] + newend + message[endq:]
    # Check for unescaped < and >
    if "<" in message or ">" in message:
        reduced = pangostrip(message)
        if "<" in reduced or ">" in reduced:
            if message == reduced:	# No pango markup
                here = message.find('<')
                if message[here:here+4] != "&lt;":
                    message = message[:here] + "&lt;" + message[here+1:]
                here = message.find('>')
                if message[here:here+4] != "&gt;":
                    message = message[:here] + "&gt;" + message[here+1:]
            else:
                print '"%s", line %d: < or > in pango string requires manual fix.' % (filename, line, oldstyle)
    return message

class WmllintIterator(WmlIterator):
    "Fold an Emacs-compatible error reporter into WmlIterator."
    def printError(self, *misc):
        """Emit an error locator compatible with Emacs compilation mode."""
        if not hasattr(self, 'lineno') or self.lineno == -1:
            print >>sys.stderr, '"%s":' % self.fname
        else:
            print >>sys.stderr, '"%s", line %d:' % (self.fname, self.lineno+1),
        for item in misc:
            print >>sys.stderr, item,
        print >>sys.stderr #terminate line

def local_sanity_check(filename, nav, key, prefix, value, comment):
    "Sanity checks that don't require file context or globals."
    errlead = '"%s", line %d: ' %  (filename, nav.lineno+1)
    ancestors = nav.ancestors()
    in_definition = "#define" in ancestors
    in_call = filter(lambda x: x.startswith("{"), ancestors)
    parent = None
    if ancestors:
        parent = ancestors[-1]
        ancestors = ancestors[:-1]
    # Check for things marked translated that aren't strings
    if "_" in nav.text and not "wmllint: ignore" in nav.text:
        m = re.search(r'[=(]\s*_\s+("?)', nav.text)
        if m and not m.group(1):
            print errlead + 'translatability mark before non-string'
    # Most tags are not allowed with [part]
    if "[part]" in ancestors and parent not in ("[part]", "[image]", "[insert_tag]", "[if]", "[then]", "[else]", "[switch]", "[case]", "[variable]", "[deprecated_message]"):
        print errlead + '%s not permitted within [part] tag' % parent
    # Most tags are not permitted inside [if]
    if len(ancestors) >= 2 and ancestors[-1] == "[if]":
        if parent not in ("[and]", "[attack_sound_frame]", "[else]", 
                          "[flame_burst_1_frame]", "[flame_burst_2_frame]",
                          "[flame_burst_3_frame]", "[flame_burst_4_frame]",
                          "[flame_burst_5_frame]", "[flame_burst_6_frame]",
                          "[frame]", "[have_location]", "[have_unit]",
                          "[missile_frame]", "[not]", "[or]",
                          "[poisoned_sound_frame]", "[sling_stone_frame]",
                          "[slowed_sound_frame]", "[then]", "[variable]"):
            print errlead + 'illegal child of [if]'
    # Check for fluky credit parts
    if parent == "[entry]":
        if key == "email" and " " in value:
            print errlead + 'space in email name'
    # Check for various things that shouldn't be outside an [ai] tag
    if not in_definition and not in_call and not "[ai]" in nav.ancestors():
        if key in ("number_of_possible_recruits_to_force_recruit",
                   "recruitment_ignore_bad_movement",
                   "recruitment_ignore_bad_combat",
                   "recruitment_pattern",
                   "villages_per_scout", "leader_value", "village_value",
                   "aggression", "caution", "attack_depth", "grouping"):
            print errlead + key + " outside [ai] scope"
    # Bad [recruit] attribute
    if parent in ("[allow_recruit]", "[disallow_recruit]") and key == "recruit":
        print errlead + "recruit= should be type="

def global_sanity_check(filename, lines):
    "Perform sanity and consistency checks on input files."
    # Sanity-check abilities and traits against notes macros.
    # Note: This check is disabled on units derived via [base_unit].
    # Also, build dictionaries of unit movement types and races
    in_unit_type = None
    notecheck = True
    trait_note = dict(notepairs)
    note_trait = dict(map(lambda p: (p[1], p[0]), notepairs))
    for nav in WmllintIterator(lines, filename):
        if nav.text.startswith("wmllint: notecheck off"):
            notecheck = False
            continue
        elif nav.text.startswith("wmllint: notecheck on"):
            notecheck = True
        #print "Element = %s, text = %s" % (nav.element, `nav.text`)
        if nav.element == "[unit_type]":
            unit_race = ""
            unit_id = ""
            base_unit = ""
            traits = []
            notes = []
            has_special_notes = False
            in_unit_type = nav.lineno+1
            hitpoints_specified = False
            continue
        elif nav.element == "[/unit_type]":
            #print '"%s", %d: unit has traits %s and notes %s' \
            #      % (filename, in_unit_type, traits, notes)
            if unit_id and base_unit:
                derived_units.append((filename, nav.lineno+1, unit_id, base_unit))
            if unit_id and not base_unit:
                missing_notes = []
                for trait in traits:
                    tn = trait_note[trait]
                    if tn not in notes and tn not in missing_notes:
                        missing_notes.append(tn)
                missing_traits = []
                for note in notes:
                    nt = note_trait[note]
                    if nt not in traits and nt not in missing_traits:
                        missing_traits.append(nt)
                if (notes or traits) and not has_special_notes:
                    missing_notes = ["{SPECIAL_NOTES}"] + missing_notes
                # If the unit didn't specify hitpoints, there is some wacky
                # stuff going on (possibly pseudo-[base_unit] behavior via
                # macro generation) so disable some of the consistency checks.
                if not hitpoints_specified:
                    continue
                if notecheck and missing_notes:
                    print '"%s", line %d: unit %s is missing notes +%s' \
                          % (filename, in_unit_type, unit_id, "+".join(missing_notes))
                if missing_traits:
                    print '"%s", line %d: unit %s is missing traits %s' \
                          % (filename, in_unit_type, unit_id, "+".join(missing_traits))
                if not (notes or traits) and has_special_notes:
                    print '"%s", line %d: unit %s has superfluous {SPECIAL_NOTES}' \
                         % (filename, in_unit_type, unit_id)
                if not "[theme]" in nav.ancestors() and not "[base_unit]" in nav.ancestors() and not unit_race:
                    print '"%s", line %d: unit %s has no race' \
                         % (filename, in_unit_type, unit_id)
            in_unit_type = None
            traits = []
            notes = []
            unit_id = ""
            base_unit = ""
            has_special_notes = False
            unit_race = None
        if '[unit_type]' in nav.ancestors() and not "[filter_attack]" in nav.ancestors():
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                if key == "id":
                    if value[0] == "_":
                        value = value[1:].strip()
                    if not unit_id and not "[base_unit]" in nav.ancestors():
                        unit_id = value
                        unit_types.append(unit_id)
                    if not base_unit and "[base_unit]" in nav.ancestors():
                        base_unit = value
                elif key == "hitpoints":
                    hitpoints_specified = True
                elif key == "usage":
                    assert(unit_id)
                    usage[unit_id] = value
                elif key == "movement_type":
                    if '{' not in value:
                        assert(unit_id)
                        unit_movetypes.append((unit_id, filename, nav.lineno+1, value))
                elif key == "race":
                    if '{' not in value:
                        assert(unit_id)
                        unit_race = value
                        unit_races.append((unit_id, filename, nav.lineno+1, unit_race))
                elif key == "advances_to":
                    assert(unit_id)
                    advancements = value
                    if advancements.strip() != "null":
                        advances.append((unit_id, filename, nav.lineno+1, advancements))
            except TypeError:
                pass
            if "{SPECIAL_NOTES}" in nav.text:
                has_special_notes = True
            for (p, q) in notepairs:
                if p in nav.text:
                    traits.append(p)
                if q in nav.text:
                    notes.append(q)
    # Detect units that speak in their death events
    filter_subject = None
    die_event = False
    for nav in WmllintIterator(lines, filename):
        if "[/event]" in nav.text:
            filter_subject = None
            die_event = False
        elif not nav.ancestors():
            continue
        elif "[event]" in nav.ancestors():
            parent = nav.ancestors()[-1]
            if parent == "[event]":
                # Check if it's a death event
                fields = parse_attribute(nav.text)
                if fields:
                    (key, prefix, value, comment) = fields 
                    if key == 'name' and value == 'die':
                        die_event = True
            elif parent == "[filter]":
                # Check to see if it has a filter subject
                if "id" in nav.text:
                    try:
                        (key,prefix,value,comment) = parse_attribute(nav.text)
                        filter_subject = value
                    except TypeError:
                        pass
            elif die_event and filter_subject and parent == "[message]":
                # Who is speaking?
                fields = parse_attribute(nav.text)
                if fields:
                    (key, prefix, value, comment) = fields 
                    if key in ("id", "speaker"):
                        if (value == filter_subject) or (value == "unit"):
                            print '"%s", line %d: %s speaks in his/her death event' % (filename, nav.lineno+1, value)
                        filter_subject = None
    # Collect information on defined movement types and races
    for nav in WmllintIterator(lines, filename):
        above = nav.ancestors()
        if above and above[-1] in ("[movetype]", "[race]"):
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                if above[-1] == "[movetype]" and key == 'name':
                    movetypes.append(value)
                if above[-1] == "[race]" and key == 'id':
                    races.append(value)
            except TypeError:
                pass
    # Sanity-check recruit and recruitment_pattern.
    # This code has a limitation; if there are multiple instances of
    # recruit and recruitment_pattern (as can happen if these lists
    # vary by EASY/NORMAL/HARD level) this code will only record the
    # last of each for later consistency checking.
    in_side = False
    in_ai = in_subunit = False
    recruit = {}
    in_generator = False
    sidecount = 0
    recruitment_pattern = {}
    ifdef_stack = [None]
    for i in range(len(lines)):
        if lines[i].startswith("#ifdef"):
            ifdef_stack.append(lines[i].strip().split()[1])
            continue
        if lines[i].startswith("#ifndef"):
            ifdef_stack.append("!" + lines[i].strip().split()[1])
            continue
        if lines[i].startswith("#else"):
            if ifdef_stack[-1].startswith("!"):
                ifdef_stack.append(ifdef_stack[-1][1:])
            else:
                ifdef_stack.append("!" + ifdef_stack[-1])
            continue
        if lines[i].startswith("#endif"):
            ifdef_stack.pop()
            continue
        if "[generator]" in lines[i]:
            in_generator = True
            continue
        elif "[/generator]" in lines[i]:
            in_generator = False
            continue
        elif "[side]" in lines[i]:
            in_side = True
            sidecount += 1
            continue
        elif "[/side]" in lines[i]:
            if recruit or recruitment_pattern:
                sides.append((filename, recruit, recruitment_pattern))
            in_side = False
            recruit = {}
            recruitment_pattern = {}
            continue
        elif in_side and "[ai]" in lines[i]:
            in_ai = True
            continue
        elif in_side and "[unit]" in lines[i]:
            in_subunit = True
            continue
        elif in_side and "[/ai]" in lines[i]:
            in_ai = False
            continue
        elif in_side and "[/unit]" in lines[i]:
            in_subunit = False
            continue
        if not in_side or in_subunit or '=' not in lines[i]:
            continue
        try:
            (key, prefix, value, comment) = parse_attribute(lines[i])
            if key == "recruit" and value:
                recruit[ifdef_stack[-1]] = (i+1, map(lambda x: x.strip(), value.split(",")))
            elif key == "recruitment_pattern" and value:
                if not in_ai:
                    print '"%s", line %d: recruitment_pattern outside [ai]' \
                              % (filename, i+1)
                else:
                    recruitment_pattern[ifdef_stack[-1]] = (i+1, map(lambda x: x.strip(), value.split(",")))
                    for utype in recruitment_pattern[ifdef_stack[-1]][1]:
                        if not utype in usage_types:
                            print '"%s", line %d: unknown usage class %s' \
                                  % (filename, i+1, utype)
            elif key == "side" and not in_ai:
                try:
                    if not in_generator and sidecount != int(value):
                        print '"%s", line %d: side number %s is out of sequence' \
                              % (filename, i+1, value)
                except ValueError:
                    pass	# Ignore ill-formed integer literals
        except TypeError:
            pass
    # Interpret various magic comments
    for i in range(len(lines)):
        # Interpret magic comments for setting the usage pattern of units.
        # This copes with some wacky UtBS units that are defined with
        # variant-spawning macros.  The prototype comment looks like this:
        #wmllint: usage of "Desert Fighter" is fighter 
        m = re.match('# *wmllint: usage of "([^"]*)" is +(.*)', lines[i])
        if m:
            usage[m.group(1)] = m.group(2).strip()
            unit_types.append(m.group(1))
        # Accumulate global spelling exceptions
        words = re.search("wmllint: general spellings? (.*)", lines[i])
        if words:
            for word in words.group(1).split():
                declared_spellings["GLOBAL"].append(word.lower())
        words = re.search("wmllint: directory spellings? (.*)", lines[i])
        if words:
            fdir = os.path.dirname(filename)
            if fdir not in declared_spellings:
                declared_spellings[fdir] = []
            for word in words.group(1).split():
                declared_spellings[fdir].append(word.lower())
    # Consistency-check the id= attributes in [side], [unit], [recall],
    # and [message] scopes, also correctness-check translation marks and look
    # for double spaces at end of sentence.
    present = []
    in_scenario = False
    in_person = False
    in_trait = False
    ignore_id = False
    in_object = False
    in_stage = False
    in_cfg = False
    in_goal = False
    in_facet = False
    in_sound_source = False
    in_remove_sound_source = False
    in_message = False
    in_option = False
    ignoreable = False
    preamble_seen = False
    sentence_end = re.compile("(?<=[.!?;:])  +")
    capitalization_error = re.compile("(?<=[.!?])  +[a-z]")
    markcheck = True
    translation_mark = re.compile(r'_ *"')
    for i in range(len(lines)):
        if '[' in lines[i]:
            preamble_seen = True
        if "[scenario]" in lines[i]:
            in_scenario = True
            preamble_seen = False
        elif "[/scenario]" in lines[i]:
            in_scenario = False
        elif "[trait]" in lines[i]:
            in_trait = True
        elif "[/trait]" in lines[i]:
            in_trait = False
        elif "[object]" in lines[i]:
            in_object = True
        elif "[/object]" in lines[i]:
            in_object = False
        elif "[stage]" in lines[i]:
            in_stage = True
        elif "[/stage]" in lines[i]:
            in_stage = False
        elif "[cfg]" in lines[i]:
            in_cfg = True
        elif "[/cfg]" in lines[i]:
            in_cfg = False
        elif "[goal]" in lines[i]:
            in_goal = True
        elif "[/goal]" in lines[i]:
            in_goal = False
        elif "[facet]" in lines[i]:
            in_facet = True
        elif "[/facet]" in lines[i]:
            in_facet = False
        elif "[sound_source]" in lines[i]:
            in_sound_source = True
        elif "[/sound_source]" in lines[i]:
            in_sound_source = False
        elif "[remove_sound_source]" in lines[i]:
            in_remove_sound_source = True
        elif "[/remove_sound_source]" in lines[i]:
            in_remove_sound_source = False
        elif "[message]" in lines[i]:
            in_message = True
        elif "[/message]" in lines[i]:
            in_message = False
        elif "[/option]" in lines[i]:
            in_option = False
        elif "[option]" in lines[i]:
            in_option = True
        elif "[label]" in lines[i] or "[chamber]" in lines[i] or "[time]" in lines[i]:
            ignore_id = True
        elif "[/label]" in lines[i] or "[/chamber]" in lines[i] or "[/time]" in lines[i]:
            ignore_id = False
        elif "[kill]" in lines[i] or "[effect]" in lines[i] or "[move_unit_fake]" in lines[i] or "[scroll_to_unit]" in lines[i]:
            ignoreable = True
        elif "[/kill]" in lines[i] or "[/effect]" in lines[i] or "[/move_unit_fake]" in lines[i] or "[/scroll_to_unit]" in lines[i]:
            ignoreable = False
        elif "[side]" in lines[i] or "[unit]" in lines[i] or "[recall]" in lines[i]:
            in_person = True
            continue
        elif "[/side]" in lines[i] or "[/unit]" in lines[i] or "[/recall]" in lines[i]:
            in_person = False
        if "wmllint: markcheck off" in lines[i]:
            markcheck = False
        if "wmllint: markcheck on" in lines[i]:
            markcheck = True
        m = re.search("# *wmllint: recognize +(.*)", lines[i])
        if m:
            present.append(string_strip(m.group(1)).strip())
        if '=' not in lines[i] or ignoreable:
            continue
        parseable = False
        try:
            (key, prefix, value, comment) = parse_attribute(lines[i])
            parseable = True
        except TypeError:
            pass
        if parseable:
            if "wmllint: ignore" in comment:
                continue
            has_tr_mark = translation_mark.search(value)
            if key == 'role':
                present.append(value)
            if has_tr_mark:
                # FIXME: This test is rather bogus as is.
                # Doing a better job would require tokenizing to pick up the
                # string boundaries. I'd do it, but AI0867 is already working
                # on a parser-based wmllint.   
                if '{' in value and "+" not in value and value.find('{') > value.find("_"):
                    print '"%s", line %d: macro reference in translatable string'\
                          % (filename, i+1)
                #if future and re.search("[.,!?]  ", lines[i]):
                #    print '"%s", line %d: extraneous space in translatable string'\
                #          % (filename, i+1)
            # Check correctness of translation marks and descriptions
            if key.startswith("#"):	# FIXME: parse_attribute is confused.
                pass
            elif key.startswith("{"):
                pass
            elif key == 'letter':	# May be led with _s for void 
                pass
            elif key in ('name', 'male_name', 'female_name'):		# FIXME: check this someday
                pass
            elif key in translatables:
                if markcheck and not value.startswith("$") and not value.startswith("{") and not re.match(" +", value) and not has_tr_mark:
                    print '"%s", line %d: %s needs translation mark' \
                          % (filename, i+1, key)
                    lines[i] = lines[i].replace('=', "=_ ")
                nv = sentence_end.sub(" ", value)
                if nv != value:
                    print '"%s", line %d: double space after sentence end' \
                          % (filename, i+1)
                    if not stringfreeze:
                        lines[i] = sentence_end.sub(" ", lines[i])
                if capitalization_error.search(lines[i]):
                    print '"%s", line %d: probable capitalization or punctuation error' \
                          % (filename, i+1)
                if key == "message" and in_message and not in_option:
                    lines[i] = pangoize(lines[i], filename, i)
            else:
                if in_scenario and key == "id":
                    if in_person:
                        present.append(value)
                    elif value in ('narrator', 'unit', 'second_unit') or (value and value[0] in ("$", "{")):
                        continue
                    elif preamble_seen and not ignore_id and not in_object and not in_cfg and not in_facet and not in_sound_source and not in_remove_sound_source and not in_stage and not in_goal and not value in present:
                        print '"%s", line %d: unknown \'%s\' referred to by id' \
                              % (filename, i+1, value)
                if markcheck and has_tr_mark and not ("wmllint: ignore" in comment or "wmllint: noconvert" in comment):
                    print '"%s", line %d: %s should not have a translation mark' \
                              % (filename, i+1, key)                
                    lines[i] = lines[i].replace("_", "", 1)
    # Now that we know who's present, register all these names as spellings
    declared_spellings[filename] = map(lambda x: x.lower(), present)
    # Check for textdomain strings; should be exactly one, on line 1
    textdomains = []
    no_text = False
    for i in range(len(lines)):
        if "#textdomain" in lines[i]:
            textdomains.append(i+1)
        elif "wmllint: no translatables":
            no_text = True
    if not no_text:
        if not textdomains:
            print '"%s", line 1: no textdomain string' % filename
        elif textdomains[0] == 1:	# Multiples are OK if first is on line 1
            pass
        elif len(textdomains) > 1:
            print '"%s", line %d: multiple textdomain strings on lines %s' % \
                  (filename, textdomains[0], ", ".join(map(str, textdomains)))
        else:
            w = textdomains[0]
            print '"%s", line %d: single textdomain declaration not on line 1.' % \
                  (filename, w)
            lines = [lines[w-1].lstrip()] + lines[:w-1] + lines[w:]
    return lines

def condition_match(p, q):
    "Do two condition-states match?"
    # The empty condition state is represented by None 
    if p is None or q is None or (p == q):
        return True
    # Past this point it's all about handling cases with negation
    sp = p
    np = False
    if sp.startswith("!"):
        sp = sp[1:]
        np = True
    sq = q
    nq = False
    if sq.startswith("!"):
        sq = sp[1:]
        nq == True
    return (sp != sq) and (np != nq)

def consistency_check():
    "Consistency-check state information picked up by sanity_check"
    derivedlist = map(lambda x: x[2], derived_units)
    baselist = map(lambda x: x[3], derived_units)
    derivations = dict(zip(derivedlist, baselist))
    for (filename, recruitdict, patterndict) in sides:
        for (rdifficulty, (rl, recruit)) in recruitdict.items():
            utypes = []
            for rtype in recruit:
                base = rtype
                if rtype not in unit_types:
                    # Assume WML coder knew what he was doing if macro reference
                    if not rtype.startswith("{"):
                        print '"%s", line %d: %s is not a known unit type' % (filename, rl, rtype)
                    continue
                elif rtype not in usage:
                    if rtype in derivedlist:
                        base = derivations[rtype]
                    else:
                        print '"%s", line %d: %s has no usage type' % \
                              (filename, rl, rtype)
                        continue
                utype = usage[base]
                utypes.append(utype)
                for (pdifficulty, (pl, recruit_pattern)) in patterndict.items():
                    if condition_match(pdifficulty, rdifficulty):
                        if utype not in recruit_pattern:
                            rshow = ''
                            if rdifficulty is not None:
                                rshow = 'At ' + rdifficulty + ', '
                            pshow = ''
                            if pdifficulty is not None:
                                pshow = ' ' + pdifficulty
                            print '"%s", line %d: %s%s (%s) doesn\'t match the%s recruitment pattern (%s) for its side' % (filename, rl, rshow, rtype, utype, pshow, ", ".join(recruit_pattern))
            # We have a list of all the usage types recruited at this sifficulty
            # in utypes.  Use it to check the matching pattern, if any. Suppress
            # this check if the recruit line is a macroexpansion.
            if recruit and not recruit[0].startswith("{"):
                for (pdifficulty, (pl, recruitment_pattern)) in patterndict.items():
                    if condition_match(pdifficulty, rdifficulty):
                        for utype in recruitment_pattern:
                            if utype not in utypes:
                                rshow = '.'
                                if rdifficulty is not None:
                                    rshow = ' at difficulty ' + rdifficulty + '.'
                                print '"%s", line %d: no %s units recruitable%s' % (filename, pl, utype, rshow)
    if movetypes:
        for (unit_id, filename, line, movetype) in unit_movetypes:
            if movetype not in movetypes:
                print '"%s", line %d: %s has unknown movement type' \
                      % (filename, line, unit_id)
    if races:
        for (unit_id, filename, line, race) in unit_races:
            if race not in races:
                print '"%s", line %d: %s has unknown race' \
                      % (filename, line, unit_id)
    # Should we be checking the transitive closure of derivation?
    # It's not clear whether [base_unit] works when the base is itself derived.
    for (filename, line, unit_type, base_unit) in derived_units:
        if base_unit not in unit_types:
            print '"%s", line %d: derivation of %s from %s does not resolve' \
                  % (filename, line, unit_type, base_unit)
    # Check that all advancements are known units
    for (unit_id, filename, lineno, advancements) in advances:
        advancements = map(string.strip, advancements.split(","))
        bad_advancements = filter(lambda x: x not in (unit_types+derivedlist), advancements)
        if bad_advancements:
            print '"%s", line %d: %s has unknown advancements %s' \
                  % (filename, lineno, unit_id, bad_advancements)

# Syntax transformations

leading_ws = re.compile(r"^\s*")

def leader(s):
    "Return a copy of the leading whitespace in the argument."
    return leading_ws.match(s).group(0)

def hack_syntax(filename, lines):
    # Syntax transformations go here.  This gets called once per WML file;
    # the name of the file is passed as filename, text of the file as the
    # array of strings in lines.  Modify lines in place as needed;
    # changes will be detected by the caller.
    #
    # Ensure that every attack has a translatable description.
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:

            break
        elif "[attack]" in lines[i]:
            j = i;
            have_description = False
            while '[/attack]' not in lines[j]:
                if lines[j].strip().startswith("description"):
                    have_description = True
                j += 1
            if not have_description:
                j = i
                while '[/attack]' not in lines[j]:
                    fields = lines[j].strip().split('#')
                    syntactic = fields[0]
                    comment = ""
                    if len(fields) > 1:
                        comment = fields[1]
                    if syntactic.strip().startswith("name"):
                        description = syntactic.split("=")[1].strip()
                        if not description.startswith('"'):
                            description = '"' + description + '"\n'
                        # Skip the insertion if this is a dummy declaration
                        # or one modifying an attack inherited from a base unit.
                        if "no-icon" not in comment:
                            new_line = leader(syntactic) + "description=_"+description
                            if verbose:
                                print '"%s", line %d: inserting %s' % (filename, i+1, `new_line`)
                            lines.insert(j+1, new_line)
                            j += 1
                    j += 1
    # Ensure that every speaker=narrator block without an image uses
    # wesnoth-icon.png as an image.
    need_image = in_message = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        precomment = lines[i].split("#")[0]
        if '[message]' in precomment:
            in_message = True
        if "speaker=narrator" in precomment:
            need_image = True
        elif precomment.strip().startswith("image"):
            need_image = False
        elif '[/message]' in precomment:
            if need_image:
                # This line presumes the code has been through wmlindent
                if verbose:
                    print '"%s", line %d: inserting "image=wesnoth-icon.png"'%(filename, i+1)
                lines.insert(i, leader(precomment) + baseindent + "image=wesnoth-icon.png\n")
            need_image = in_message = False
    # Hack tracking-map macros from 1.4 and earlier.  The idea is to lose
    # all assumptions about colors in the names
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].startswith("#"):
            pass
        elif "{DOT_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("DOT_CENTERED", "NEW_JOURNEY")    
        elif "{DOT_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("DOT_WHITE_CENTERED", "OLD_JOURNEY")
        elif "{CROSS_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("CROSS_CENTERED", "NEW_BATTLE")    
        elif "{CROSS_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("CROSS_WHITE_CENTERED", "OLD_BATTLE")
        elif "{FLAG_RED_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("FLAG_RED_CENTERED", "NEW_REST")    
        elif "{FLAG_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("FLAG_WHITE_CENTERED", "OLD_REST")
        elif "{DOT " in lines[i] or "CROSS" in lines[i]:
            m = re.search("{(DOT|CROSS) ([0-9]+) ([0-9]+)}", lines[i])
            if m:
                n = m.group(1)
                if n == "DOT":
                    n = "NEW_JOURNEY"
                if n == "CROSS":
                    n = "NEW_BATTLE"
                x = int(m.group(2)) + 5
                y = int(m.group(3)) + 5
                lines[i] = lines[i][:m.start(0)] +("{%s %d %d}" % (n, x, y)) + lines[i][m.end(0):]
    # Fix bare strings containing single quotes; these confuse wesnoth-mode.el
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        elif lines[i].count("'") % 2 == 1:
            try:
                (key, prefix, value, comment) = parse_attribute(lines[i])
                if  "'" in value and value[0].isalpha() and value[-1].isalpha() and not '"'+value+'"' in lines[i]:
                    newtext = prefix + '"' + value + '"' + comment + "\n"
                    if lines[i] != newtext:
                        lines[i] = newtext
                        if verbose:
                            print '"%s", line %d: quote-enclosing attribute value.'%(filename, i+1)
            except TypeError:
                pass
    # Palette transformation for 1.7:
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].startswith("#"):
            pass
        # RC -> PAL
        elif "RC" in lines[i]:
            lines[i] = re.sub(r"~RC\(([^=\)]*)=([^)]*)\)",r"~PAL(\1>\2)",lines[i])
    # Rename the terrain definition tag
    in_standing_anim = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].startswith("#"):
            pass
        # Ugh...relies on code having been wmlindented
        lines[i] = re.sub(r"^\[terrain\]", "[terrain_type]", lines[i])
        lines[i] = re.sub(r"^\[/terrain\]", "[/terrain_type]", lines[i])
        if "[standing_anim]" in lines[i]:
            in_standing_anim = True
        if "[/standing_anim]" in lines[i]:
            in_standing_anim = False
        if in_standing_anim:
            lines[i] = re.sub(r"terrain([^_])", r"terrain_type\1", lines[i])
    # campaigns directory becomes add-ons
    in_binary_path = in_textdomain = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].startswith("#"):
            pass
        # The trouble with this transformation is that it's only right for UMC;
        # it clobbers mainline.
        if 0:
            if "[binary_path]" in lines[i]:
                in_binary_path = True
            if "[/binary_path]" in lines[i]:
                in_binary_path = False
            if "[textdomain]" in lines[i]:
                in_textdomain = True
            if "[/textdomain]" in lines[i]:
                in_textdomain = False
            if in_binary_path or in_textdomain:
                lines[i] = re.sub(r"data/campaigns", r"data/add_ons", lines[i])
        # This is done on every line
        if "campaigns/" in lines[i]:
            lines[i] = lines[i].replace("{~campaigns/", "{~add-ons/")
            lines[i] = lines[i].replace("{@campaigns/", "{~add-ons/")
        elif "@add-ons/" in lines[i]:
            lines[i] = lines[i].replace("{@add-ons/", "{~add-ons/")
    # More syntax transformations would go here.
    return lines

# Generic machinery starts here

def is_map(filename):
    "Is this file a map?"
    return filename.endswith(".map")

if 0:	# Not used, as there are currently no defined map transforms
    class maptransform_error:
        "Error object to be thrown by maptransform."
        def __init__(self, infile, inline, type):
            self.infile = infile
            self.inline = inline
            self.type = type
        def __repr__(self):
            return '"%s", line %d: %s' % (self.infile, self.inline, self.type)

    def maptransform_sample(filename, baseline, inmap, y):
        "Transform a map line."
        # Sample to illustrate how map-transformation hooks are called.
        # The baseline argument will be the starting line number of the map.
        # The inmap argument will be a 2D string array containing the
        # entire map.  y will be the vertical coordinate of the map line.
        # You pass a list of these as the second argument of translator().
        raise maptransform_error(filename, baseline+y+1,
                             "unrecognized map element at line %d" % (y,))

tagstack = []	# For tracking tag nesting

def outermap(func, inmap):
    "Apply a transformation based on neighborhood to the outermost ring."
    # Top and bottom rows
    for i in range(len(inmap[0])):
        inmap[0][i] = func(inmap[0][i])
        inmap[len(inmap)-1][i] = func(inmap[len(inmap)-1][i])
    # Leftmost and rightmost columns excluding top and bottom rows
    for i in range(1, len(inmap)-1):
        inmap[i][0] = func(inmap[i][0])
        inmap[i][len(inmap[0])-1] = func(inmap[i][len(inmap[0])-1])

def translator(filename, mapxforms, textxform):
    "Apply mapxform to map lines and textxform to non-map lines."
    global tagstack
    gzipped = filename.endswith(".gz")
    if gzipped:
        unmodified = gzip.open(filename).readlines()
    else:
        unmodified = file(filename).readlines()
    # Pull file into an array of lines, CR-stripping as needed
    mfile = []
    map_only = filename.endswith(".map")
    terminator = "\n"
    for line in unmodified:
        if line.endswith("\n"):
            line = line[:-1]
        if line.endswith("\r"):
            line = line[:-1]
            if not stripcr:
                terminator = '\r\n'
        mfile.append(line)
        if "map_data" in line:
            map_only = False
    # Process line-by-line
    lineno = baseline = 0
    cont = False
    validate = True
    unbalanced = False
    newdata = []
    refname = None
    while mfile:
        if not map_only:
            line = mfile.pop(0)
            if verbose >= 3:
                sys.stdout.write(line + terminator)
            lineno += 1
        # Check for one certain error condition
        if line.count("{") and line.count("}"):
            refname = line[line.find("{"):line.rfind("}")]            
            # Ignore all-caps macro arguments.
            if refname == refname.upper():
                pass
            elif 'mask=' in line and not (refname.endswith("}") or refname.endswith(".mask")):
                print \
                      '"%s", line %d: fatal error, mask file without .mask extension (%s)' \
                      % (filename, lineno+1, refname)
                sys.exit(1)
        # Exclude map_data= lines that are just 1 line without
        # continuation, or which contain {}.  The former are
        # pathological and the parse won't handle them, the latter
        # refer to map files which will be checked separately.
        if map_only or (("map_data=" in line or "mask=" in line)
                        and line.count('"') in (1, 2)
                        and line.count("{") == 0
                        and  line.count("}") == 0
                        and not within('time')):
            outmap = []
            add_border = True
            add_usage = True
            have_header = have_delimiter = False
            maskwarn = False
            maptype = None
            if map_only:
                if filename.endswith(".mask"):
                    maptype = "mask"
                else:
                    maptype = "map"
            else:
                leadws = leader(line)
                if "map_data" in line:
                    maptype = "map"
                elif "mask" in line:
                    maptype = "mask"
            baseline = lineno
            cont = True
            if not map_only:
                fields = line.split('"')
                if fields[1].strip():
                    mfile.insert(0, fields[1])
                if len(fields) == 3:
                    mfile.insert(1, '"')
            if verbose >= 3:
                print "*** Entering %s mode on:" % maptype
                print mfile
            # Gather the map header (if any) and data lines
            savedheaders = []
            while cont and mfile:
                line = mfile.pop(0)
                if verbose >= 3:
                    sys.stdout.write(line + terminator)
                lineno += 1
                # This code supports ignoring comments and header lines
                if len(line) == 0 or line[0] == '#' or '=' in line:
                    if '=' in line:
                        have_header = True
                    if 'border_size' in line:
                        add_border = False
                    if "usage" in line:
                        add_usage = False
                        usage = line.split("=")[1].strip()
                        if usage == 'mask':
                            add_border = False
                            if filename.endswith(".map"):
                                print "warning: usage=mask in file with .map extension"
                        elif usage == 'map':
                            if filename.endswith(".mask"):
                                print "warning: usage=map in file with .mask extension"
                    if len(line) == 0:
                        have_delimiter = True
                    savedheaders.append(line + terminator)
                    continue
                if '"' in line:
                    cont = False
                    if verbose >= 3:
                        print "*** Exiting map mode."
                    line = line.split('"')[0]
                if line:
                    if ',' in line:
                        fields = line.split(",")
                    else:
                        fields = map(lambda x: x, line)
                    outmap.append(fields)
                    if not maskwarn and maptype == 'map' and "_f" in line:
                        print \
                              '"%s", line %d: warning, fog in map file' \
                              % (filename, lineno+1)
                        maskwarn = True
            # Checking the outmap length here is a bit of a crock;
            # the one-line map we don't want to mess with is in the
            # NO_MAP macro.
            if len(outmap) == 1:
                add_border = add_usage = False
            # Deduce the map type
            if not map_only:
                if maptype == "map":
                    newdata.append(leadws + "map_data=\"")
                elif maptype == "mask":
                    newdata.append(leadws + "mask=\"")
            original = copy.deepcopy(outmap)
            for transform in mapxforms:
                for y in range(len(outmap)):
                    transform(filename, baseline, outmap, y)
            if maptype == "mask":
                add_border = False
            if add_border:
                print '%s, "line %d": adding map border...' % \
                      (filename, baseline)
                newdata.append("border_size=1" + terminator)
                have_header = True
                # Start by duplicating the current outermost ring
                outmap = [outmap[0]] + outmap + [outmap[-1]]
                for i in range(len(outmap)):
                    outmap[i] = [outmap[i][0]] + outmap[i] + [outmap[i][-1]]
                # Strip villages out of the edges
                outermap(lambda n: re.sub(r"\^V[a-z]+", "", n), outmap) 
                # Strip keeps out of the edges
                outermap(lambda n: re.sub(r"K([a-z]+)", r"C\1", n), outmap)
                # Strip the starting positions out of the edges
                outermap(lambda n: re.sub(r"[1-9] ", r"", n), outmap)
                # Turn big trees on the edges to ordinary forest hexes
                outermap(lambda n: n.replace(r"Gg^Fet", r"Gs^Fp"), outmap)
            if add_usage:
                print '%s, "line %d": adding %s usage header...' % \
                      (filename, baseline, maptype)
                newdata.append("usage=" + maptype + terminator)
                have_header = True
            newdata += savedheaders
            if have_header and not have_delimiter:
                newdata.append(terminator)
            for y in range(len(outmap)):
                newdata.append(",".join(outmap[y]) + terminator)
            # All lines of the map are processed, add the appropriate trailer
            if not map_only:
                newdata.append("\"" + terminator)
        elif "map_data=" in line and (line.count("{") or line.count("}")):
            newline = line
            refre = re.compile(r"\{@?([^A-Z].*)\}").search(line)
            if refre:
                mapfile = refre.group(1)
                if not mapfile.endswith(".map") and is_map(mapfile):
                    newline = newline.replace(mapfile, mapfile + ".map") 
            newdata.append(newline + terminator)
            if newline != line:
                if verbose > 0:
                    print 'wmllint: "%s", line %d: %s -> %s.' % (filename, lineno, line, newline)
        elif "map_data=" in line and line.count('"') > 1:
            print 'wmllint: "%s", line %d: one-line map.' % (filename, lineno)
            newdata.append(line + terminator)
        else:
            # Handle text (non-map) lines.  It can use within().
            newline = textxform(filename, lineno, line)
            newdata.append(newline + terminator)
            fields = newline.split("#")
            trimmed = fields[0]
            destringed = re.sub('"[^"]*"', '', trimmed)	# Ignore string literals
            comment = ""
            if len(fields) > 1:
                comment = fields[1]
            # Now do warnings based on the state of the tag stack.
            if not unbalanced:
                for instance in re.finditer(r"\[\/?\+?([a-z][a-z_]*[a-z])\]", destringed):
                    tag = instance.group(1)
                    attributes = []
                    closer = instance.group(0)[1] == '/'
                    if not closer:
                        tagstack.append((tag, {}))
                    else:
                        if len(tagstack) == 0:
                            print '"%s", line %d: closer [/%s] with tag stack empty.' % (filename, lineno+1, tag) 
                        elif tagstack[-1][0] != tag:
                            print '"%s", line %d: unbalanced [%s] closed with [/%s].' % (filename, lineno+1, tagstack[-1][0], tag)
                        else:
                            if validate:
                                validate_on_pop(tagstack, tag, filename, lineno)
                            tagstack.pop()
                if tagstack:
                    for instance in re.finditer(r'([a-z][a-z_]*[a-z])\s*=(.*)', trimmed):
                        attribute = instance.group(1)
                        value = instance.group(2)
                        if '#' in value:
                            value = value.split("#")[0]
                        tagstack[-1][1][attribute] = value.strip()
            if "wmllint: validate-on" in comment:
                validate = True
            if "wmllint: validate-off" in comment:
                validate = False
            if "wmllint: unbalanced-on" in comment:
                unbalanced = True
            if "wmllint: unbalanced-off" in comment:
                unbalanced = False
            if "wmllint: match" in comment:
                comment = comment.strip()
                fields = comment.split()
                if len(fields) == 5 and fields[3] == "with":
                    notepairs.append((fields[2], fields[4]))
    # It's an error if the tag stack is nonempty at the end of any file:
    if tagstack:
        print '"%s", line %d: tag stack nonempty (%s) at end of file.' % (filename, lineno, tagstack)
    tagstack = []
    if iswml(filename):
        # Perform checks that are purel local.  This is an
        # optimization hack to reduce parsing overhead.
        for nav in WmllintIterator(newdata, filename):
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                local_sanity_check(filename, nav, key, prefix, value, comment)
            except TypeError:
                pass
        # Perform file-global semantic sanity checks
        newdata = global_sanity_check(filename, newdata)
        # OK, now perform WML rewrites
        newdata = hack_syntax(filename, newdata)
        # Run everything together
        filetext = "".join(newdata)
        transformed = filetext
    else:
        # Map or mask -- just run everything together
        transformed = "".join(newdata)
    # Simple check for unbalanced macro calls
    unclosed = None
    linecount = 1
    startline = None
    quotecount = 0
    display_state = False
    singleline = False
    for i in range(len(transformed)):
        if transformed[i] == '\n':
            if singleline:
                singleline = False
		if not display_state and quotecount % 2 and transformed[i:i+2] != "\n\n" and transformed[i-1:i+1] != "\n\n":
                    print '"%s", line %d: nonstandard word-wrap style within message' % (filename, linecount)
            linecount += 1
        elif transformed[i-7:i] == "message" and not transformed[i] == ']':
            singleline = True
        elif re.match(" *wmllint: *display +on", transformed[i:]):
            display_state = True
        elif re.match(" *wmllint: *display +off", transformed[i:]):
            display_state = False
        elif transformed[i] == '"':
            quotecount += 1
            if quotecount % 2 == 0:
                singleline = False
    # Return None if the transformation functions made no changes.
    if "".join(unmodified) != transformed:
        return transformed
    else:
        return None

def inner_spellcheck(nav, value, spelldict):
    "Spell-check an attribute value or string."
    # Strip off translation marks
    if value.startswith("_"):
        value = value[1:].strip()
    # Strip off line continuations, they interfere with string-stripping
    value = value.strip()
    if value.endswith("+"):
        value = value[:-1].rstrip()
    # Strip off string quotes
    value = string_strip(value)
    # Discard extraneous stuff 
    value = value.replace("...", " ")
    value = value.replace("/", " ")
    value = value.replace(")", " ")
    value = value.replace("(", " ")
    value = value.replace("\xe2\x80\xa6", " ")	# UTF-8 ellipsis
    value = value.replace("\xe2\x80\x94", " ")	# UTF-8 em dash
    value = value.replace("\xe2\x80\x93", " ")	# UTF-8 en dash
    value = value.replace("\xe2\x80\x95", " ")	# UTF-8 horizontal dash
    value = value.replace("''", "")
    value = value.replace("female^", " ")
    value = value.replace("male^", " ")
    value = value.replace("teamname^", " ")
    value = value.replace("team_name^", " ")
    value = value.replace("weapon^", " ")
    value = value.replace("UI^", " ")
    if '<' in value:
        value = re.sub("<[^>]+>text='([^']*)'<[^>]+>", r"\1", value)
        value = re.sub("<[0-9,]+>", "", value)
    # Fold continued lines
    value = re.sub(r'" *\+\s*_? *"', "", value)
    # It would be nice to use pyenchant's tokenizer here, but we can't
    # because it wants to strip the trailing quotes we need to spot
    # the Dwarvish-accent words.
    for token in value.split():
        # Try it with simple lowercasing first
        lowered = token.lower()
        if d.check(lowered):
            continue
        # Strip leading punctuation and grotty Wesnoth highlighters
        # Last char in this regexp is to ignore concatenation signs.
        while lowered and lowered[0] in " \t(`@*'%_+":
            lowered = lowered[1:]
        # Not interested in interpolations or numeric literals
        if not lowered or lowered.startswith("$"):
            continue
        # Suffix handling. Done in two passes because some
        # Dwarvish dialect words end in a single quote
        while lowered and lowered[-1] in "_-*).,:;?!& \t":
            lowered = lowered[:-1]
        if lowered and spelldict.check(lowered):
            continue;
        while lowered and lowered[-1] in "_-*').,:;?!& \t":
            lowered = lowered[:-1]
        # Not interested in interpolations or numeric literals
        if not lowered or lowered.startswith("$") or lowered[0].isdigit():
            continue
       # Nuke balanced string quotes if present
        lowered = string_strip(lowered)
        if lowered and spelldict.check(lowered):
            continue
        # No match? Strip posessive suffixes and try again.
        elif lowered.endswith("'s") and spelldict.check(lowered[:-2]):
            continue
        # Hyphenated compounds need all their parts good
        if "-" in lowered:
            parts = lowered.split("-")
            if filter(lambda w: not w or spelldict.check(w), parts) == parts:
                continue
        # Modifier literals aren't interesting
        if re.match("[+-][0-9]", lowered):
            continue
        # Match various onomatopoetic exclamations of variable form
        if re.match("hm+", lowered):
            continue
        if re.match("a+[ur]*g+h*", lowered):
            continue
        if re.match("(mu)?ha(ha)*", lowered):
            continue
        if re.match("ah+", lowered):
            continue
        if re.match("no+", lowered):
            continue
        if re.match("no+", lowered):
            continue
        if re.match("um+", lowered):
            continue
        if re.match("aw+", lowered):
            continue
        if re.match("o+h+", lowered):
            continue
        if re.match("s+h+", lowered):
            continue
        nav.printError('possible misspelling "%s"' % token)
                

def spellcheck(fn, d):
    "Spell-check a file using an Enchant dictionary object."
    local_spellings = []
    # Accept declared spellings for this file
    # and for all directories above it.
    up = fn
    while True:
        if not up or up == os.sep:
            break
        else:
            local_spellings += declared_spellings.get(up,[])
            up = os.path.dirname(up)
    local_spellings = filter(lambda w: not d.check(w), local_spellings)
    #if local_spellings:
    #    print "%s: inherited local spellings: %s" % (fn, local_spellings)
    map(d.add_to_session, local_spellings)

    # Process this individual file
    for nav in WmllintIterator(filename=fn):
        #print "element=%s, text=%s" % (nav.element, `nav.text`)
        # Recognize local spelling exceptions
        if not nav.element and "#" in nav.text:
            comment = nav.text[nav.text.index("#"):]
            words = re.search("wmllint: local spellings? (.*)", comment)
            if words:
                for word in words.group(1).split():
                    word = word.lower()
                    if not d.check(word):
                        d.add_to_session(word)
                        local_spellings.append(word)
                    else:
                        nav.printError(" %s already declared" % word)
    #if local_spellings:
    #    print "%s: with this file's local spellings: %s" % (fn,local_spellings)

    for nav in WmllintIterator(filename=fn):
        # Spell-check message and story parts
        if nav.element in spellcheck_these:
            # Special case, beyond us until we can do better filtering..
            # There is lots of strange stuff in text- attributes in the
            # helpfile(s).
            if nav.element == 'text=' and '[help]' in nav.ancestors():
                continue
            # Remove pango markup
            if "<" in nav.text or ">" in nav.text or '&' in nav.text:
                nav.text = pangostrip(nav.text)
            # Spell-check the attribute value
            (key, prefix, value, comment) = parse_attribute(nav.text)
            if "no spellcheck" in comment:
                continue
            inner_spellcheck(nav, value, d)
        # Take exceptions from the id fields
        if nav.element == "id=":
            (key, prefix, value, comment) = parse_attribute(nav.text)
            value = string_strip(value).lower()
            if value and not d.check(value):
                d.add_to_session(value)
                local_spellings.append(value)
    #if local_spellings:
    #    print "%s: slated for removal: %s" % (fn, local_spellings)
    for word in local_spellings:
        try:
            d.remove_from_session(word)
        except AttributeError:
            print "Caught AttributeError when trying to remove %s from dict" % word

vctypes = (".svn", ".git")

def interesting(fn):
    "Is a file interesting for conversion purposes?"
    return fn.endswith(".cfg") or is_map(fn) or issave(fn)

def allcfgfiles(dir):
    "Get the names of all interesting files under dir."
    datafiles = []
    if not os.path.isdir(dir):
        if interesting(dir):
            if not os.path.exists(dir):
                sys.stderr.write("wmllint: %s does not exist\n" % dir)
            else:
                datafiles.append(dir)
    else:
        for root, dirs, files in os.walk(dir):
            for vcsubdir in vctypes: 
                if vcsubdir in dirs:
                    dirs.remove(vcsubdir)
            for name in files:
                if interesting(os.path.join(root, name)):
                    datafiles.append(os.path.join(root, name))
    datafiles.sort()	# So diffs for same campaigns will cluster in reports
    return map(os.path.normpath, datafiles)

def help():
        sys.stderr.write("""\
Usage: wmllint [options] [dir]
    Convert Battle of Wesnoth WML from older versions to newer ones.
    Takes any number of directories as arguments.  Each directory is converted.
    If no directories are specified, acts on the current directory.
    Options may be any of these:
    -h, --help                 Emit this help message and quit.
    -d, --dryrun               List changes but don't perform them.
    -v, --verbose              -v        lists changes.
                               -v -v     names each file before it's processed.
                               -v -v -v  shows verbose parse details.
    -c, --clean                Clean up -bak files.
    -D, --diff                 Display diffs between converted and unconverted files.
    -r, --revert               Revert the conversion from the -bak files.
    -s, --stripcr              Convert DOS-style CR/LF to Unix-style LF.
    -f, --future               Enable experimental WML conversions.
    -S, --nospellcheck         Suppress spellchecking
    -Z, --stringfreeze         Suppress warnings about newlines in messages
""")

if __name__ == '__main__':
    try:
        (options, arguments) = getopt.getopt(sys.argv[1:], "cdDfhnrsvSZ", [
            "clean",
            "diffs",
            "dryrun",
            "future",
            "help",
            "revert",
            "stripcr",
            "verbose",
            "nospellcheck",
            ])
    except getopt.GetoptError:
        help()
        sys.exit(1)
    clean = False
    diffs = False
    dryrun = False
    future = False
    revert = False
    stringfreeze = False
    stripcr = False
    verbose = 0
    dospellcheck = True
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        elif switch in ('-c', '--clean'):
            clean = True
        elif switch in ('-d', '--dryrun'):
            dryrun = True
            verbose = max(1, verbose)
        elif switch in ('-D', '--diffs'):
            diffs = True
        elif switch in ('-f', '--future'):
            future = True
        elif switch in ('-r', '--revert'):
            revert = True
        elif switch in ('-s', '--stripcr'):
            stripcr = True
        elif switch in ('-Z', '--stringfreeze'):
            stringfreeze = True
        elif switch in ('-v', '--verbose'):
            verbose += 1
        elif switch in ('-S', '--nospellcheck'):
            dospellcheck = False
    if clean and revert:
        sys.stderr.write("wmllint: can't do clean and revert together.\n")
        sys.exit(1)

    post15 = False

    def hasdigit(str):
        for c in str:
            if c in "0123456789":
                return True
        return False

    def texttransform(filename, lineno, line):
        "Resource-name transformation on text lines."
        original = line
        # Perform line changes
        if "wmllint: noconvert" not in original:
            for (old, new) in linechanges:
                line = line.replace(old, new)
        # Perform tag renaming for 1.5.  Note: this has to happen before
        # the sanity check, which assumes [unit] has already been
        # mapped to [unit_type].  Also, beware that this test will fail to
        # convert any unit definitions not in conventionally-named
        # directories -- this is necessary in order to avoid stepping
        # on SingleUnitWML in macro files.  The post15 flag expresses whether
        # we've seen a [unit_type] and can therefore assume the files have
        # undergone 1.4 -> 1.5 conversion.
        global post15
        if "units" in filename and not post15:
            if '[unit_type]' in line:
                post15 = True
            else:
                line = line.replace("[unit]", "[unit_type]")
                line = line.replace("[+unit]", "[+unit_type]")
                line = line.replace("[/unit]", "[/unit_type]")
        # Handle SingleUnitWML or Standard Unit Filter or SideWML
        # Also, when macro calls have description= in them, the arg is
        # a SUF being passed in.
        if tagstack and ((under("unit") and not "units" in filename) or \
               standard_unit_filter() or \
               under("side") or \
               re.search("{[A-Z]+.*description=.*}", line)):
            if "id" not in tagstack[-1][1] and "_" not in line:
                line = re.sub(r"\bdescription\s*=", "id=", line)
            if "name" not in tagstack[-1][1]:
                line = re.sub(r"user_description\s*=", "name=", line)
        # Now, inside objects...
        if under("object") and "description" not in tagstack[-1][1]:
            line = re.sub(r"user_description\s*=", "description=", line)
        # Alas, WML variable references cannot be converted so
        # automatically.
        if ".description" in line: 
            print '"%s", line %d: .description may need hand fixup' % \
                                   (filename, lineno)
        if ".user_description" in line: 
            print '"%s", line %d: .user_description may need hand fixup' % \
                                   (filename, lineno)
        # In unit type definitions
        if under("unit_type") or under("female") or under("unit"):
            line = line.replace("unit_description=", "description=")
            line = line.replace("advanceto=", "advances_to=")
        # Inside themes
        if within("theme"):
            line = line.replace("[unit_description]", "[unit_name]")
        # Report the changes
        if verbose > 0 and line != original:
            msg = "%s, line %d: %s -> %s" % \
                  (filename, lineno, original.strip(), line.strip())
            print msg
        return line

    try:
        if not arguments:
            arguments = ["."]

        for dir in arguments:
            ofp = None
            for fn in allcfgfiles(dir):
                if verbose >= 2:
                    print fn + ":"
                backup = fn + "-bak"
                if clean or revert:
                    # Do housekeeping
                    if os.path.exists(backup):
                        if clean:
                            print "wmllint: removing %s" % backup
                            if not dryrun:
                                os.remove(backup)
                        elif revert:
                            print "wmllint: reverting %s" % backup
                            if not dryrun:
                                os.rename(backup, fn)
                elif diffs:
                    # Display diffs
                    if os.path.exists(backup):
                        fromdate = time.ctime(os.stat(backup).st_mtime)
                        todate = time.ctime(os.stat(fn).st_mtime)
                        fromlines = open(backup, 'U').readlines()
                        tolines = open(fn, 'U').readlines()
                        diff = difflib.unified_diff(fromlines, tolines,
                                             backup, fn, fromdate, todate, n=3)
                        sys.stdout.writelines(diff)
                else:
                    if "~" in fn:
                        print "wmllint: ignoring %s, the campaign server won't accept it." % fn
                        continue
                    # Do file conversions
                    try:
                        changed = translator(fn, [], texttransform)
                        if changed:
                            print "wmllint: converting", fn
                            if not dryrun:
                                os.rename(fn, backup)
                                if fn.endswith(".gz"):
                                    ofp = gzip.open(fn, "w")
                                    ofp.write(changed)
                                    ofp.close()
                                else:
                                    ofp = open(fn, "w")
                                    ofp.write(changed)
                                    ofp.close()
                    #except maptransform_error, e:
                    #    sys.stderr.write("wmllint: " + `e` + "\n")
                    except:
                        sys.stderr.write("wmllint: internal error on %s\n" % fn)
                        (exc_type, exc_value, exc_traceback) = sys.exc_info()
                        raise exc_type, exc_value, exc_traceback
        if not clean and not diffs and not revert:
            # Consistency-check everything we got from the file scans
            consistency_check()
            # Attempt a spell-check
            if dospellcheck:
                try:
                    import enchant
                    d = enchant.Dict("en_US")
                    checker = d.provider.desc
                    if checker.endswith(" Provider"):
                        checker = checker[:-9]
                    print "# Spell-checking with", checker
                    for word in declared_spellings["GLOBAL"]:
                        d.add_to_session(word.lower())
                    for dir in arguments:
                        ofp = None
                        for fn in allcfgfiles(dir):
                            if verbose >= 2:
                                print fn + ":"
                            spellcheck(fn, d)
                except ImportError:
                    sys.stderr.write("wmllint: spell check unavailable, install python-enchant to enable\n")
    except KeyboardInterrupt:
        print "Aborted"

# wmllint ends here
