
^Yc           @   s9  d  Z  d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z y d d l Z Wn e	 k
 r e
 Z n Xd d l m Z d d l m Z d d l m Z d d l m Z e j e  Z e e j g  Z d e f d     YZ d	 e f d
     YZ d   Z d   Z d   Z d   Z d S(   s$   Checker Manager and Checker classes.iN(   t   defaults(   t
   exceptions(   t	   processor(   t   utilst   Managerc           B   s   e  Z d  Z d   Z d   Z d   Z d   Z d   Z d d  Z	 d   Z
 d   Z d	   Z d
   Z d   Z d d  Z d   Z RS(   sR  Manage the parallelism and checker instances for each plugin and file.

    This class will be responsible for the following:

    - Determining the parallelism of Flake8, e.g.:

      * Do we use :mod:`multiprocessing` or is it unavailable?

      * Do we automatically decide on the number of jobs to use or did the
        user provide that?

    - Falling back to a serial way of processing files if we run into an
      OSError related to :mod:`multiprocessing`

    - Organizing the results of each checker so we can group the output
      together and make our output deterministic.
    c         C   s   | |  _  | |  _ | j |  _ | |  _ |  j   |  _ |  j d k |  _ d |  _ g  |  _	 g  |  _
 i d d 6d d 6d d 6d d 6|  _ |  j r y t j |  j t  |  _ Wq t k
 r } | j t k r   n  t |  _ q Xn  d S(   s  Initialize our Manager instance.

        :param style_guide:
            The instantiated style guide for this instance of Flake8.
        :type style_guide:
            flake8.style_guide.StyleGuide
        :param list arguments:
            The extra arguments parsed from the CLI (if any)
        :param checker_plugins:
            The plugins representing checks parsed from entry-points.
        :type checker_plugins:
            flake8.plugins.manager.Checkers
        i   i    t   filess   logical liness   physical linest   tokensN(   t	   argumentst   style_guidet   optionst   checkst
   _job_countt   jobst   using_multiprocessingt   Nonet   poolt	   processest   checkerst
   statisticst   multiprocessingt   Poolt
   _pool_initt   OSErrort   errnot   SERIAL_RETRY_ERRNOSt   False(   t   selfR   R   t   checker_pluginst   oserr(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   __init__9   s*    							c         C   sb   x? |  j  D]4 } x+ t j D]  } |  j | c | j | 7<q Wq
 W|  j d c t |  j   7<d  S(   NR   (   R   R    t   STATISTIC_NAMESR   t   len(   R   t   checkert	   statistic(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   _process_statistics_   s    "c         C   s   t  s t j d  d St j   rA t j   rA t j d  d St j |  j  rd t j d  d S|  j j	 r t j d  d S|  j j
 } | d k r | j   r t j d |  d S| d k r y t  j   SWq t k
 r d SXn  t |  S(   NsG   The multiprocessing module is not available. Ignoring --jobs arguments.i    s   The --jobs option is not available on Windows due to a bug (https://bugs.python.org/issue27649) in Python 2.7.11+ and 3.3+. We have detected that you are running an unsupported version of Python on Windows. Ignoring --jobs arguments.s]   The --jobs option is not compatible with supplying input using - . Ignoring --jobs arguments.sc   The --diff option was specified with --jobs but they are not compatible. Ignoring --jobs arguments.t   autos]   "%s" is not a valid parameter to --jobs. Must be one of "auto" or a numerical value, e.g., 4.(   R   t   LOGt   warningR   t
   is_windowst"   can_run_multiprocessing_on_windowst   is_using_stdinR   R	   t   diffR   t   isdigitt	   cpu_countt   NotImplementedErrort   int(   R   R   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   e   s0    	c   
      C   sg   |  j  } d } xQ | D]I \ } } } } }	 | | j d | d | d | d | d | d |	  7} q W| S(   Ni    t   codet   filenamet   line_numbert   column_numbert   textt   physical_line(   R   t   handle_error(
   R   R/   t   resultsR   t   reported_results_countt
   error_codeR0   t   columnR2   R3   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   _handle_results   s    	c         C   s   | d k r1 |  j  j d k r" t S|  j  j } n  |  j  j } | sG t St j j |  } t j | |  r t	 j
 d |  t St j j |  } t j | |  } t	 j
 d | | r d n d  | S(   s  Check if a path is excluded.

        :param str path:
            Path to check against the exclude patterns.
        :returns:
            True if there are exclude patterns and the path matches,
            otherwise False.
        :rtype:
            bool
        t   -t   stdins   "%s" has been excludeds   "%s" has %sbeen excludedt    s   not (   R	   t   stdin_display_nameR   t   excludet   ost   patht   basenameR   t   fnmatchR$   t   debugt   Truet   abspath(   R   R@   R>   RA   t   absolute_patht   match(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   is_path_excluded   s     c            s   | d k r  j } n  | s* d g } n   j j   j j    f d     j j         f d   | D } g  | D] } | j r | ^ q  _ t	 j
 d t  j   d S(   s   Create checkers for each file.t   .c            sY   t  j |     } |  d k } t j j |   }  o@ | |  k } | rU | pX | pX | S(   s-   Determine if we should create a file checker.R:   (   R   RB   R?   R@   t   exists(   R/   t   argumentt   matches_filename_patternst   is_stdint   file_existst   explicitly_provided(   t   filename_patternst   running_from_vcs(    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   should_create_file_checker   s    c         3   sO   |  ]E } t  j |  j  D]) }  | |  r t |    j  Vq q d  S(   N(   R   t   filenames_fromRH   t   FileCheckerR	   (   t   .0RK   R/   (   R
   R   RR   (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pys	   <genexpr>   s   	s   Checking %d filesN(   R   R   R	   R/   t   _running_from_vcsR
   t   to_dictionaryt   should_processR   R$   t   infoR   (   R   t   pathsR   R    (    (   R
   RP   RQ   R   RR   s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   make_checkers   s    
%c      	   C   s   d } } xq |  j  D]f } t | j d d   } | j } |  j j |   | |  j | |  7} Wd QX| t |  7} q W| | f S(   s4  Report all of the errors found in the managed file checkers.

        This iterates over each of the checkers and reports the errors sorted
        by line number.

        :returns:
            A tuple of the total results found and the results reported.
        :rtype:
            tuple(int, int)
        i    t   keyc         S   s   |  d |  d f S(   Ni   i   (    (   t   tup(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   <lambda>  s    N(   R   t   sortedR5   t   display_nameR   t   processing_fileR9   R   (   R   t   results_reportedt   results_foundR    R5   R/   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   report   s    
	c         C   s0   |  j  d  k	 r, |  j  j   |  j  j   n  d  S(   N(   R   R   t	   terminatet   join(   R   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   _force_cleanup  s    c   	      C   s   t  j t  } t  j t  } |  j j t |  j d t t	 |  j  |  j
  } x1 | D]) } | \ } } } | | | <| | | <qX W|  j j   |  j j   d |  _ xC |  j D]8 } | j } t | | d d   | _ | | | _ q Wd S(   s   Run the checkers in parallel.t	   chunksizeR\   c         S   s   |  d |  d f S(   Ni   (    (   R]   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR^   )  s    N(   t   collectionst   defaultdictt   listt   dictR   t   imap_unorderedt   _run_checksR   t   calculate_pool_chunksizeR   R   t   closeRf   R   R`   R_   R5   R   (	   R   t   final_resultst   final_statisticst   pool_mapt   retR/   R5   R   R    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   run_parallel  s(    		
		c         C   s"   x |  j  D] } | j   q
 Wd S(   s   Run the checkers in serial.N(   R   t
   run_checks(   R   R    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt
   run_serial,  s    c         C   s   z y$ |  j  r |  j   n
 |  j   Wn{ t k
 rx } | j t k r[ t j |    n  t j d |  |  j   n- t	 k
 r t j d  t
 j d   n XWd |  j   Xd S(   sH  Run all the checkers.

        This will intelligently decide whether to run the checks in parallel
        or whether to run them in serial.

        If running the checks in parallel causes a problem (e.g.,
        https://gitlab.com/pycqa/flake8/issues/74) this also implements
        fallback to serial processing.
        s(   Running in serial after OS exception, %rs"   Flake8 was interrupted by the users   Early quit while running checksN(   R   Ru   Rw   R   R   R   R$   t	   exceptionR%   t   KeyboardInterruptR   t	   EarlyQuitRg   (   R   R   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   run1  s    
 	c         C   s   t  j d  |  j |  d S(   s   Start checking files.

        :param list paths:
            Path names to check. This is passed directly to
            :meth:`~Manager.make_checkers`.
        s   Making checkersN(   R$   RY   R[   (   R   RZ   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   startL  s    c         C   s?   |  j    x. |  j D]# } t j d | j  | j   q Wd S(   s   Stop checking files.s   Joining %s to the main processN(   R"   R   R$   RY   t   nameRf   (   R   t   proc(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   stopV  s    
N(   t   __name__t
   __module__t   __doc__R   R"   R   R9   RH   R   R[   Rd   Rg   Ru   Rw   R{   R|   R   (    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   &   s   	&		7		/					
RT   c           B   s   e  Z d  Z d   Z d   Z d d  Z d   Z e d    Z	 d   Z
 d   Z d d  Z d	   Z d
   Z d   Z d   Z d   Z RS(   s;   Manage running checks for a file and aggregate the results.c         C   s   | |  _  | |  _ | |  _ g  |  _ i d d 6d d 6d d 6|  _ |  j   |  _ | |  _ t |  _	 |  j d k	 r |  j j |  _ |  j j   |  _	 t |  j j  |  j d <n  d S(   sh  Initialize our file checker.

        :param str filename:
            Name of the file to check.
        :param checks:
            The plugins registered to check the file.
        :type checks:
            dict
        :param options:
            Parsed option values from config and command-line.
        :type options:
            optparse.Values
        i    R   s   logical liness   physical linesN(   R	   R/   R
   R5   R   t   _make_processorR   R`   R   RX   R   t   should_ignore_fileR   t   lines(   R   R/   R
   R	   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   a  s    						c         C   st   y t  j |  j |  j  SWnS t k
 ro t j   d  \ } } d j | j |  } |  j	 d d d |  d  SXd  S(   Ni   s   {0}: {1}t   E902i    (   R   t   FileProcessorR/   R	   t   IOErrort   syst   exc_infot   formatR   Rd   R   (   R   t   exc_typeRx   t   message(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR     s    c         C   s   | d k r' | j d d  \ } } n  | } | r[ t |  d d  r[ |  j j |  } n  | | | | | f } |  j j |  | S(   s2   Report an error by storing it in the results list.t    i   R   N(   R   t   splitt   getattrR   t   line_forR5   t   append(   R   R7   R0   R8   R2   t   lineR3   t   error(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyRd     s    c         K   sw   t  j d | |  y |  j j | d |  Wn8 t k
 rh } t  j d  t j d | d |   n X| d |   S(   s!   Run the check in a single plugin.s   Running %r with %rt
   parameterss$   Plugin requested unknown parameters.t   pluginRx   (   R$   RC   R   t   keyword_arguments_fort   AttributeErrorR   R   t    PluginRequestedUnknownParameters(   R   R   R   t   ae(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt	   run_check  s    		c   
      C   s  d } t  |  j  d k rS |  j d } t  |  d k r_ | d d !\ } } q_ n d \ } } | d k r| rt |  t  rd } d } | d } | d  k	 r | j d  j d  } t  |  d } | d } t  |  }	 | |	 k r |	 } q n  | | 8} | | 8} n  | | f S(	   Ni   i   i   i    is   
(    (   i   i    (   R   t   argst
   isinstancet   SyntaxErrorR   t   rstripR   (
   Rx   t   tokent   rowR8   t   column_offsett
   row_offsetR3   R   t   logical_linet   logical_line_length(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   _extract_syntax_information  s(    !


c         C   s   y |  j  j   } Wnp t t t f k
 r t j   d  \ } } |  j |  \ } } |  j d | | d | j	 | j
 d f  d SXx |  j d D] } |  j | d | } y | j   } Wn t k
 r | } n Xx< | D]4 \ }	 }
 } } |  j d d d	 |	 d
 |
 d |  q Wq Wd S(   s1   Run all checks expecting an abstract syntax tree.i   t   E999s   %s: %si    Nt   ast_pluginst   treeR7   R0   R8   R2   (   R   t	   build_astt
   ValueErrorR   t	   TypeErrorR   R   R   Rd   R   R   R
   R   R{   R   R   (   R   t   astR   Rx   R   R8   R   R    t   runnerR0   t   offsetR2   t   check(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   run_ast_checks  s(    
	c   
      C   s   |  j  j   \ } } } | s" d S|  j  j |  t j d | j    x |  j d D] } |  j  j |  |  j | d | p d	 } xQ | D]I \ } } t	 | |  } | \ } }	 |  j
 d d d | d |	 d |  q WqV W|  j  j   d S(
   s(   Run all checks expecting a logical line.Ns   Logical line: "%s"t   logical_line_pluginsR   R7   R0   R8   R2   (    (   R   t   build_logical_linet   update_stateR$   RC   R   R
   t   update_checker_state_forR   t   find_offsetRd   R   t   next_logical_line(
   R   t   commentsR   t   mappingR   R5   R   R2   R0   R   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   run_logical_checks  s"    	c         C   s   x |  j  d D] } |  j j |  |  j | d | } | d k	 r | \ } } |  j d d d |  j j d | d | d | p |  } |  j j | |  q q Wd S(	   s)   Run all checks for a given physical line.t   physical_line_pluginsR3   R7   R0   R8   R2   R   N(   R
   R   R   R   R   Rd   R0   t   check_physical_error(   R   R3   t   override_error_lineR   t   resultR   R2   R7   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   run_physical_checks  s    	c         C   s&  d } |  j  } |  j } x | j   D] } | d c d 7<|  j |  | d d !\ } } t j t |  | t j k r t j | |  } q% | d k r% t j	 |  r |  j
 |  q t j |  r t | j  d k r |  j | |  q q% q% W| j r"|  j | j d  |  j   n  d S(   s   Process tokens and trigger checks.

        This can raise a :class:`flake8.exceptions.InvalidSyntax` exception.
        Instead of using this directly, you should use
        :meth:`flake8.checker.FileChecker.run_checks`.
        i    R   i   i   iN(   R   R   t   generate_tokenst   check_physical_eolt	   log_tokenR$   t   tokenizet   OPt   count_parenthesest   token_is_newlinet   handle_newlinet   token_is_commentR   R   t   handle_commentR   R   R   (   R   t   parensR   t   file_processorR   t
   token_typeR2   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   process_tokens  s&    			c         C   s   y |  j    Wn8 t j k
 rH } |  j | j | j | j | j  n X|  j   |  j	 j
 d } | |  j
 d <|  j |  j |  j
 f S(   s   Run checks against the file.s   logical lines(   R   R   t   InvalidSyntaxRd   R7   R0   R1   t   error_messageR   R   R   R/   R5   (   R   t   exct   logical_lines(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyRv   5  s    
c         C   sp   t  |  } | j d  | d <| d d | d d t | d  f | d <t |  g |  j _ |  j   d S(   s3   Handle the logic when encountering a comment token.s   
i   i   i    i   N(   Rk   R   R   t   tupleR   R   R   (   R   R   t
   token_text(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   C  s
    .c         C   sl   | t  j k r) |  j   |  j j   n? t |  j j  d k r^ |  j j   |  j j   n
 |  j   d S(   s3   Handle the logic when encountering a newline token.i   N(	   R   t   NEWLINER   R   t   reset_blank_beforeR   R   t   visited_new_blank_linet   delete_first_token(   R   R   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   L  s    
c      
   C   s   t  j |  r# |  j | d  nt t  j |  r | d d } |  j  j d |  = x5 |  j  j |  D]! } |  j | d d | d qi WWd QXn  d S(   s@   Run physical checks if and only if it is at the end of the line.i   i   i    R0   s   
R   N(   R   t   is_eol_tokenR   t   is_multiline_stringt   inside_multilinet
   split_line(   R   R   t   line_noR   (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   X  s    N(   R   R   R   R   R   R   Rd   R   t   staticmethodR   R   R   R   R   Rv   R   R   R   (    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyRT   ^  s   			%							c           C   s   t  j  t  j t  j  d S(   s:   Ensure correct signaling of ^C using multiprocessing.Pool.N(   t   signalt   SIGINTt   SIG_IGN(    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR   p  s    c         C   s   t  |  | d d  S(   s  Determine the chunksize for the multiprocessing Pool.

    - For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap  # noqa
    - This formula, while not perfect, aims to give each worker two batches of
      work.
    - See: https://gitlab.com/pycqa/flake8/merge_requests/156#note_18878876
    - See: https://gitlab.com/pycqa/flake8/issues/265
    i   i   (   t   max(   t   num_checkerst   num_jobs(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyRo   u  s    	c         C   s
   |  j    S(   N(   Rv   (   R    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyRn     s    c         C   sT   t  |  t  r |  Sx$ | D] \ } } |  | k r Pq q W| d | d |  | f S(   s*   Find the offset tuple for a single offset.i    i   (   R   R   (   R   R   t   token_offsett   position(    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyR     s    (   R   Ri   R   t   loggingR?   R   R   R   R   t   ImportErrorR   t   flake8R    R   R   R   t	   getLoggerR   R$   t   sett   ENOSPCR   t   objectR   RT   R   Ro   Rn   R   (    (    (    s.   /tmp/pip-build-EndXZ2/flake8/flake8/checker.pyt   <module>   s4   
 9 			