diff --git a/.gitignore b/.gitignore index b2a399d..a80596b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,12 @@ rdoc spec/reports test/tmp test/version_tmp +test/data/icdar-groundtruth tmp /*.pdf /*.csv +*.class +*.prefs # YARD artifacts .yardoc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e69de29 diff --git a/Gemfile b/Gemfile index 5c6cdb3..5aa14bf 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,3 @@ -source "http://rubygems.org" +source "https://rubygems.org" gemspec gem "rake" diff --git a/README.md b/README.md index 82c83b9..9633aff 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,20 @@ -tabula-extractor -================ +tabula-extractor (old version) +============================== -[![Build Status](https://travis-ci.org/jazzido/tabula-extractor.png)](https://travis-ci.org/jazzido/tabula-extractor) +**Deprecation Note:** *This is the old version of the Tabula extraction engine. New projects wishing to integrate Tabula should use [tabula-java][tabula-java] (the new Java version of this extraction engine) unless you prefer to use JRuby. Users looking for the command-line version of Tabula should also use [tabula-java][tabula-java].* + +[tabula-java]: http://www.github.com/tabulapdf/tabula-java + +--- + +Extract tables from PDF files. `tabula-extractor` is the table extraction engine that used to power [Tabula](http://tabula.nerdpower.org). + +If you're beginning a new project, consider using [tabula-java](http://www.github.com/tabulapdf/tabula-java), a pure-Java version of the extraction engine behind Tabula. If you want Ruby bindings and are okay using JRuby (or have already begin a project), you may continue to use this project. This project's JRuby backend has been replaced with the Java backend; all that remains here is a thin wrapper for Ruby compatibility. This wrapper maintains API backwards-compatibility with the old, pure-JRuby implementation that we all know and love. -Extract tables from PDF files. `tabula-extractor` is the table extraction engine that powers [Tabula](http://tabula.nerdpower.org), now available as a library and command line program. ## Installation -At the moment, `tabula-extractor` only works with JRuby. [Install JRuby](http://jruby.org/getting-started) and run +`tabula-extractor` only works with JRuby 1.7 or newer. [Install JRuby](http://jruby.org/getting-started) and run `` jruby -S gem install tabula-extractor @@ -57,12 +64,12 @@ Here's a very basic example: ````ruby require 'tabula' - + pdf_file_path = "whatever.pdf" outfilename = "whatever.csv" - + out = open(outfilename, 'w') - + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, :all ) extractor.extract.each do |pdf_page| pdf_page.spreadsheets.each do |spreadsheet| @@ -73,7 +80,3 @@ end out.close ```` - -## Notes - -`tabula-extractor` uses [LSD: a Line Segment Detector](http://www.ipol.im/pub/art/2012/gjmr-lsd/) by Rafael Grompone von Gioi, Jérémie Jakubowicz, Jean-Michel Morel and Gregory Randall. diff --git a/Rakefile b/Rakefile index 3b96779..dd5d838 100755 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,6 @@ #!/usr/bin/env rake require 'bundler' require 'rake' -#require 'rake/testtask' Bundler::GemHelper.install_tasks @@ -9,4 +8,10 @@ task :test do ruby %{-X+C -J-Xmx512m test/tests.rb} end +task :compile do + Dir.chdir(File.join(File.dirname(__FILE__), 'ext/tabula')) do + system("mvn", "clean", "compile", "assembly:single", out: $stdout, err: $stderr) + end +end + task :default => [:test] diff --git a/bin/tabula b/bin/tabula index de3499c..9c0be9d 100755 --- a/bin/tabula +++ b/bin/tabula @@ -1,5 +1,6 @@ #!/usr/bin/env jruby -J-Djava.awt.headless=true # encoding: utf-8 + require 'trollop' require_relative '../lib/tabula' @@ -22,12 +23,12 @@ def parse_pages_arg(pages_arg) pages += (s.to_i..e.to_i).to_a end end - pages.sort + pages.sort.map { |i| i.to_java(:int) } end def parse_command_line opts = Trollop::options do - version "tabula #{Tabula::VERSION} (c) 2012-2013 Manuel Aristarán" + version "tabula #{Tabula::VERSION} (c) 2012-2016 Manuel Aristarán, Jeremy B. Merrill, Mike Tigas and other contributors" banner <<-EOS Tabula helps you extract tables from PDFs @@ -101,8 +102,9 @@ def main false end - extractor = Tabula::Extraction::ObjectExtractor.new(filename, parse_pages_arg(opts[:pages]), opts[:password]) - extractor.extract.each_with_index do |pdf_page, page_index| + extractor = Tabula::Extraction::ObjectExtractor.new(filename, password=opts[:password]) + extractor.extract(parse_pages_arg(opts[:pages])) + .each_with_index do |pdf_page, page_index| #do the heuristic here if use_spreadsheet_extraction.nil? @@ -115,25 +117,24 @@ def main if use_spreadsheet_extraction if opts[:debug] pdf_page.spreadsheets.each do |spreadsheet| - STDERR.puts "Page #{pdf_page.number(:one_indexed)}: #{spreadsheet.dims(:top, :left, :bottom, :right)}" + STDERR.puts "Page #{pdf_page.number(:one_indexed)}: #{spreadsheet}" end end - tables = pdf_page.spreadsheets(:use_line_returns=> use_line_returns).map(&:rows) + tables = pdf_page.spreadsheets(:use_line_returns=> use_line_returns) else STDERR.puts "Page #{pdf_page.number(:one_indexed)}: #{page_area.to_s}" if opts[:debug] if opts[:guess] - page_areas = pdf_page.spreadsheets.map{|rect| pdf_page.get_area(rect.dims(:top, :left, :bottom, :right))} + page_areas = pdf_page.spreadsheets.map{|rect| pdf_page.get_area(rect)} elsif area_input - page_areas = [pdf_page.get_area(area_input)] + page_areas = [pdf_page.get_area(*area_input)] else page_areas = [pdf_page] end - tables = page_areas.map{|page_area| page_area.make_table(vertical_rulings.nil? ? {} : { :vertical_rulings => rulings_from_columns(pdf_page, page_area, vertical_rulings) })} + tables = page_areas.map{|page_area| page_area.get_table(vertical_rulings.nil? ? {} : { :vertical_rulings => rulings_from_columns(pdf_page, page_area, vertical_rulings) })} end + tables.each do |table| - Tabula::Writers.send(opts[:format].to_sym, - table, - out) + out << table.send("to_#{opts[:format].downcase}".to_sym) end end out.close diff --git a/ext/COPYING b/ext/COPYING deleted file mode 100644 index dba13ed..0000000 --- a/ext/COPYING +++ /dev/null @@ -1,661 +0,0 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. diff --git a/ext/Makefile.OSX b/ext/Makefile.OSX deleted file mode 100644 index bd69167..0000000 --- a/ext/Makefile.OSX +++ /dev/null @@ -1,18 +0,0 @@ -include Makefile.defaults - - -CFLAGS := -arch i386 -arch x86_64 -fPIC -O3 -g -Wall -Werror - -lib: lib$(NAME).$(VERSION).dylib - -lib$(NAME).$(VERSION).dylib: $(NAME).o - $(CC) -arch i386 -arch x86_64 -dynamiclib -lm -o lib$(NAME).dylib $^ - -clean: - $(RM) *.o - -$(NAME)_test: lib$(NAME).$(VERSION).dylib - $(CC) lsd_call_example.c -o $@ -L. -l$(NAME) - -test: $(NAME)_test - LD_LIBRARY_PATH=. ./$(NAME)_test diff --git a/ext/Makefile.defaults b/ext/Makefile.defaults deleted file mode 100644 index 82599ad..0000000 --- a/ext/Makefile.defaults +++ /dev/null @@ -1,9 +0,0 @@ -CFLAGS := -fPIC -O3 -g -Wall -Werror -CC := gcc -MAJOR := 1 -MINOR := 0 -NAME := lsd -VERSION := $(MAJOR).$(MINOR) - -clean: - $(RM) *.o *.so* *.dylib diff --git a/ext/Makefile.linux32 b/ext/Makefile.linux32 deleted file mode 100644 index 1c0bdc5..0000000 --- a/ext/Makefile.linux32 +++ /dev/null @@ -1,11 +0,0 @@ -include Makefile.defaults - -# link statically with musl-libc -CC = /home/manuel/tabula-build/musl-32/bin/musl-gcc -CFLAGS := -fPIC -Wall -Werror - -lib: lib$(NAME).$(VERSION).so - -lib$(NAME).$(VERSION).so: $(NAME).o - $(CC) -shared -static -o lib$(NAME)-linux32.so $^ - diff --git a/ext/Makefile.linux64 b/ext/Makefile.linux64 deleted file mode 100644 index d600b55..0000000 --- a/ext/Makefile.linux64 +++ /dev/null @@ -1,12 +0,0 @@ -# to compile a x86_64 lib in an ubuntu i386 box -include Makefile.defaults - -# link statically with musl-libc -CC = /home/manuel/tabula-build/musl-64/bin/musl-gcc -CFLAGS := -fPIC -Wall -Werror -m64 - -lib: lib$(NAME).$(VERSION).so - -lib$(NAME).$(VERSION).so: $(NAME).o - @LDEMULATION=elf_x86_64 $(CC) -m64 -shared -static -o lib$(NAME)-linux64.so $^ - diff --git a/ext/Makefile.mingw b/ext/Makefile.mingw deleted file mode 100644 index 8a1fb4f..0000000 --- a/ext/Makefile.mingw +++ /dev/null @@ -1,10 +0,0 @@ -include Makefile.defaults - -#CC = /usr/local/gcc-4.8.0-qt-4.8.4-for-mingw32/win32-gcc/bin/i586-mingw32-gcc -CC = /usr/bin/i686-w64-mingw32-gcc-4.6 -CFLAGS := -Wall -Werror - -lib: lib$(NAME).$(VERSION).dll - -lib$(NAME).$(VERSION).dll: $(NAME).o - $(CC) -shared -o lib$(NAME).dll liblsd.def $^ diff --git a/ext/Makefile.mingw64 b/ext/Makefile.mingw64 deleted file mode 100644 index baa923f..0000000 --- a/ext/Makefile.mingw64 +++ /dev/null @@ -1,10 +0,0 @@ -include Makefile.defaults - -#CC = /usr/local/gcc-4.8.0-qt-4.8.4-for-mingw32/win32-gcc/bin/i586-mingw32-gcc -CC = /usr/bin/x86_64-w64-mingw32-gcc -CFLAGS := -Wall -Werror - -lib: lib$(NAME).$(VERSION).dll - -lib$(NAME).$(VERSION).dll: $(NAME).o - $(CC) -shared -o lib$(NAME)64.dll liblsd.def $^ diff --git a/ext/liblsd-linux32.so b/ext/liblsd-linux32.so deleted file mode 100755 index a7371eb..0000000 Binary files a/ext/liblsd-linux32.so and /dev/null differ diff --git a/ext/liblsd-linux64.so b/ext/liblsd-linux64.so deleted file mode 100755 index 2d3b202..0000000 Binary files a/ext/liblsd-linux64.so and /dev/null differ diff --git a/ext/liblsd.def b/ext/liblsd.def deleted file mode 100644 index 31b828d..0000000 --- a/ext/liblsd.def +++ /dev/null @@ -1,3 +0,0 @@ -EXPORTS -lsd -free_values diff --git a/ext/liblsd.dll b/ext/liblsd.dll deleted file mode 100755 index 86333ac..0000000 Binary files a/ext/liblsd.dll and /dev/null differ diff --git a/ext/liblsd.dylib b/ext/liblsd.dylib deleted file mode 100755 index a025ed1..0000000 Binary files a/ext/liblsd.dylib and /dev/null differ diff --git a/ext/liblsd64.dll b/ext/liblsd64.dll deleted file mode 100755 index 28b9804..0000000 Binary files a/ext/liblsd64.dll and /dev/null differ diff --git a/ext/lsd.c b/ext/lsd.c deleted file mode 100644 index ea8a2ad..0000000 --- a/ext/lsd.c +++ /dev/null @@ -1,2270 +0,0 @@ -/*---------------------------------------------------------------------------- - - LSD - Line Segment Detector on digital images - - This code is part of the following publication and was subject - to peer review: - - "LSD: a Line Segment Detector" by Rafael Grompone von Gioi, - Jeremie Jakubowicz, Jean-Michel Morel, and Gregory Randall, - Image Processing On Line, 2012. DOI:10.5201/ipol.2012.gjmr-lsd - http://dx.doi.org/10.5201/ipol.2012.gjmr-lsd - - Copyright (c) 2007-2011 rafael grompone von gioi - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - Additional permission under GNU GPL version 3 section 7 - - If you modify this Program, or any covered work, by linking or - combining it with Tabula (or a modified version of that library), - containing parts covered by the terms of "MIT License", the - licensors of this Program grant you additional permission to convey - the resulting work. Corresponding Source for a non-source form of - such a combination shall include the source code for the parts of - Tabula used as well as that of the covered work. - - - ----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** @file lsd.c - LSD module code - @author rafael grompone von gioi - */ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** @mainpage LSD code documentation - - This is an implementation of the Line Segment Detector described - in the paper: - - "LSD: A Fast Line Segment Detector with a False Detection Control" - by Rafael Grompone von Gioi, Jeremie Jakubowicz, Jean-Michel Morel, - and Gregory Randall, IEEE Transactions on Pattern Analysis and - Machine Intelligence, vol. 32, no. 4, pp. 722-732, April, 2010. - - and in more details in the CMLA Technical Report: - - "LSD: A Line Segment Detector, Technical Report", - by Rafael Grompone von Gioi, Jeremie Jakubowicz, Jean-Michel Morel, - Gregory Randall, CMLA, ENS Cachan, 2010. - - The version implemented here includes some further improvements - described in the following publication, of which this code is part: - - "LSD: a Line Segment Detector" by Rafael Grompone von Gioi, - Jeremie Jakubowicz, Jean-Michel Morel, and Gregory Randall, - Image Processing On Line, 2012. DOI:10.5201/ipol.2012.gjmr-lsd - http://dx.doi.org/10.5201/ipol.2012.gjmr-lsd - - The module's main function is lsd(). - - The source code is contained in two files: lsd.h and lsd.c. - - HISTORY: - - version 1.6 - nov 2011: - - changes in the interface, - - max_grad parameter removed, - - the factor 11 was added to the number of test - to consider the different precision values - tested, - - a minor bug corrected in the gradient sorting - code, - - the algorithm now also returns p and log_nfa - for each detection, - - a minor bug was corrected in the image scaling, - - the angle comparison in "isaligned" changed - from < to <=, - - "eps" variable renamed "log_eps", - - "lsd_scale_region" interface was added, - - minor changes to comments. - - version 1.5 - dec 2010: Changes in 'refine', -W option added, - and more comments added. - - version 1.4 - jul 2010: lsd_scale interface added and doxygen doc. - - version 1.3 - feb 2010: Multiple bug correction and improved code. - - version 1.2 - dec 2009: First full Ansi C Language version. - - version 1.1 - sep 2009: Systematic subsampling to scale 0.8 and - correction to partially handle "angle problem". - - version 1.0 - jan 2009: First complete Megawave2 and Ansi C Language - version. - - @author rafael grompone von gioi - */ -/*----------------------------------------------------------------------------*/ - -#include -#include -#include -#include -#include -#include "lsd.h" - -/** ln(10) */ -#ifndef M_LN10 -#define M_LN10 2.30258509299404568402 -#endif /* !M_LN10 */ - -/** PI */ -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif /* !M_PI */ - -#ifndef FALSE -#define FALSE 0 -#endif /* !FALSE */ - -#ifndef TRUE -#define TRUE 1 -#endif /* !TRUE */ - -/** Label for pixels with undefined gradient. */ -#define NOTDEF -1024.0 - -/** 3/2 pi */ -#define M_3_2_PI 4.71238898038 - -/** 2 pi */ -#define M_2__PI 6.28318530718 - -/** Label for pixels not used in yet. */ -#define NOTUSED 0 - -/** Label for pixels already used in detection. */ -#define USED 1 - -/*----------------------------------------------------------------------------*/ -/** Chained list of coordinates. - */ -struct coorlist -{ - int x,y; - struct coorlist * next; -}; - -/*----------------------------------------------------------------------------*/ -/** A point (or pixel). - */ -struct point {int x,y;}; - - -/*----------------------------------------------------------------------------*/ -/*------------------------- Miscellaneous functions --------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Fatal error, print a message to standard-error output and exit. - */ -static void error(char * msg) -{ - fprintf(stderr,"LSD Error: %s\n",msg); - exit(EXIT_FAILURE); -} - -/*----------------------------------------------------------------------------*/ -/** Doubles relative error factor - */ -#define RELATIVE_ERROR_FACTOR 100.0 - -/*----------------------------------------------------------------------------*/ -/** Compare doubles by relative error. - - The resulting rounding error after floating point computations - depend on the specific operations done. The same number computed by - different algorithms could present different rounding errors. For a - useful comparison, an estimation of the relative rounding error - should be considered and compared to a factor times EPS. The factor - should be related to the cumulated rounding error in the chain of - computation. Here, as a simplification, a fixed factor is used. - */ -static int double_equal(float a, float b) -{ - float abs_diff,aa,bb,abs_max; - - /* trivial case */ - if( a == b ) return TRUE; - - abs_diff = fabs(a-b); - aa = fabs(a); - bb = fabs(b); - abs_max = aa > bb ? aa : bb; - - /* DBL_MIN is the smallest normalized number, thus, the smallest - number whose relative error is bounded by DBL_EPSILON. For - smaller numbers, the same quantization steps as for DBL_MIN - are used. Then, for smaller numbers, a meaningful "relative" - error should be computed by dividing the difference by DBL_MIN. */ - if( abs_max < DBL_MIN ) abs_max = DBL_MIN; - - /* equal if relative error <= factor x eps */ - return (abs_diff / abs_max) <= (RELATIVE_ERROR_FACTOR * DBL_EPSILON); -} - -/*----------------------------------------------------------------------------*/ -/** Computes Euclidean distance between point (x1,y1) and point (x2,y2). - */ -static float dist(float x1, float y1, float x2, float y2) -{ - return sqrt( (x2-x1)*(x2-x1) + (y2-y1)*(y2-y1) ); -} - - -/*----------------------------------------------------------------------------*/ -/*----------------------- 'list of n-tuple' data type ------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** 'list of n-tuple' data type - - The i-th component of the j-th n-tuple of an n-tuple list 'ntl' - is accessed with: - - ntl->values[ i + j * ntl->dim ] - - The dimension of the n-tuple (n) is: - - ntl->dim - - The number of n-tuples in the list is: - - ntl->size - - The maximum number of n-tuples that can be stored in the - list with the allocated memory at a given time is given by: - - ntl->max_size - */ -typedef struct ntuple_list_s -{ - unsigned int size; - unsigned int max_size; - unsigned int dim; - float * values; -} * ntuple_list; - -/*----------------------------------------------------------------------------*/ -/** Free memory used in n-tuple 'in'. - */ -static void free_ntuple_list(ntuple_list in) -{ - if( in == NULL || in->values == NULL ) - error("free_ntuple_list: invalid n-tuple input."); - free( (void *) in->values ); - free( (void *) in ); -} - -/*----------------------------------------------------------------------------*/ -/** Create an n-tuple list and allocate memory for one element. - @param dim the dimension (n) of the n-tuple. - */ -static ntuple_list new_ntuple_list(unsigned int dim) -{ - ntuple_list n_tuple; - - /* check parameters */ - if( dim == 0 ) error("new_ntuple_list: 'dim' must be positive."); - - /* get memory for list structure */ - n_tuple = (ntuple_list) malloc( sizeof(struct ntuple_list_s) ); - if( n_tuple == NULL ) error("not enough memory."); - - /* initialize list */ - n_tuple->size = 0; - n_tuple->max_size = 1; - n_tuple->dim = dim; - - /* get memory for tuples */ - n_tuple->values = (float *) malloc( dim*n_tuple->max_size * sizeof(float) ); - if( n_tuple->values == NULL ) error("not enough memory."); - - return n_tuple; -} - -/*----------------------------------------------------------------------------*/ -/** Enlarge the allocated memory of an n-tuple list. - */ -static void enlarge_ntuple_list(ntuple_list n_tuple) -{ - /* check parameters */ - if( n_tuple == NULL || n_tuple->values == NULL || n_tuple->max_size == 0 ) - error("enlarge_ntuple_list: invalid n-tuple."); - - /* duplicate number of tuples */ - n_tuple->max_size *= 2; - - /* realloc memory */ - n_tuple->values = (float *) realloc( (void *) n_tuple->values, - n_tuple->dim * n_tuple->max_size * sizeof(float) ); - if( n_tuple->values == NULL ) error("not enough memory."); -} - -/*----------------------------------------------------------------------------*/ -/** Add a 7-tuple to an n-tuple list. - */ -static void add_7tuple( ntuple_list out, float v1, float v2, float v3, - float v4, float v5, float v6, float v7 ) -{ - /* check parameters */ - if( out == NULL ) error("add_7tuple: invalid n-tuple input."); - if( out->dim != 7 ) error("add_7tuple: the n-tuple must be a 7-tuple."); - - /* if needed, alloc more tuples to 'out' */ - if( out->size == out->max_size ) enlarge_ntuple_list(out); - if( out->values == NULL ) error("add_7tuple: invalid n-tuple input."); - - /* add new 7-tuple */ - out->values[ out->size * out->dim + 0 ] = v1; - out->values[ out->size * out->dim + 1 ] = v2; - out->values[ out->size * out->dim + 2 ] = v3; - out->values[ out->size * out->dim + 3 ] = v4; - out->values[ out->size * out->dim + 4 ] = v5; - out->values[ out->size * out->dim + 5 ] = v6; - out->values[ out->size * out->dim + 6 ] = v7; - - /* update number of tuples counter */ - out->size++; -} - - -/*----------------------------------------------------------------------------*/ -/*----------------------------- Image Data Types -----------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** char image data type - - The pixel value at (x,y) is accessed by: - - image->data[ x + y * image->xsize ] - - with x and y integer. - */ -typedef struct image_char_s -{ - unsigned char * data; - unsigned int xsize,ysize; -} * image_char; - -/*----------------------------------------------------------------------------*/ -/** Free memory used in image_char 'i'. - */ -static void free_image_char(image_char i) -{ - if( i == NULL || i->data == NULL ) - error("free_image_char: invalid input image."); - free( (void *) i->data ); - free( (void *) i ); -} - -/*----------------------------------------------------------------------------*/ -/** Create a new image_char of size 'xsize' times 'ysize'. - */ -static image_char new_image_char(unsigned int xsize, unsigned int ysize) -{ - image_char image; - - /* check parameters */ - if( xsize == 0 || ysize == 0 ) error("new_image_char: invalid image size."); - - /* get memory */ - image = (image_char) malloc( sizeof(struct image_char_s) ); - if( image == NULL ) error("not enough memory."); - image->data = (unsigned char *) calloc( (size_t) (xsize*ysize), - sizeof(unsigned char) ); - if( image->data == NULL ) error("not enough memory."); - - /* set image size */ - image->xsize = xsize; - image->ysize = ysize; - - return image; -} - -/*----------------------------------------------------------------------------*/ -/** Create a new image_char of size 'xsize' times 'ysize', - initialized to the value 'fill_value'. - */ -static image_char new_image_char_ini( unsigned int xsize, unsigned int ysize, - unsigned char fill_value ) -{ - image_char image = new_image_char(xsize,ysize); /* create image */ - unsigned int N = xsize*ysize; - unsigned int i; - - /* check parameters */ - if( image == NULL || image->data == NULL ) - error("new_image_char_ini: invalid image."); - - /* initialize */ - for(i=0; idata[i] = fill_value; - - return image; -} - -/*----------------------------------------------------------------------------*/ -/** int image data type - - The pixel value at (x,y) is accessed by: - - image->data[ x + y * image->xsize ] - - with x and y integer. - */ -typedef struct image_int_s -{ - int * data; - unsigned int xsize,ysize; -} * image_int; - -/*----------------------------------------------------------------------------*/ -/** Create a new image_int of size 'xsize' times 'ysize'. - */ -static image_int new_image_int(unsigned int xsize, unsigned int ysize) -{ - image_int image; - - /* check parameters */ - if( xsize == 0 || ysize == 0 ) error("new_image_int: invalid image size."); - - /* get memory */ - image = (image_int) malloc( sizeof(struct image_int_s) ); - if( image == NULL ) error("not enough memory."); - image->data = (int *) calloc( (size_t) (xsize*ysize), sizeof(int) ); - if( image->data == NULL ) error("not enough memory."); - - /* set image size */ - image->xsize = xsize; - image->ysize = ysize; - - return image; -} - -/*----------------------------------------------------------------------------*/ -/** Create a new image_int of size 'xsize' times 'ysize', - initialized to the value 'fill_value'. - */ -static image_int new_image_int_ini( unsigned int xsize, unsigned int ysize, - int fill_value ) -{ - image_int image = new_image_int(xsize,ysize); /* create image */ - unsigned int N = xsize*ysize; - unsigned int i; - - /* initialize */ - for(i=0; idata[i] = fill_value; - - return image; -} - -/*----------------------------------------------------------------------------*/ -/** double image data type - - The pixel value at (x,y) is accessed by: - - image->data[ x + y * image->xsize ] - - with x and y integer. - */ -typedef struct image_double_s -{ - float * data; - unsigned int xsize,ysize; -} * image_double; - -/*----------------------------------------------------------------------------*/ -/** Free memory used in image_double 'i'. - */ -static void free_image_double(image_double i) -{ - if( i == NULL || i->data == NULL ) - error("free_image_double: invalid input image."); - free( (void *) i->data ); - free( (void *) i ); -} - -/*----------------------------------------------------------------------------*/ -/** Create a new image_double of size 'xsize' times 'ysize'. - */ -static image_double new_image_double(unsigned int xsize, unsigned int ysize) -{ - image_double image; - - /* check parameters */ - if( xsize == 0 || ysize == 0 ) error("new_image_double: invalid image size."); - - /* get memory */ - image = (image_double) malloc( sizeof(struct image_double_s) ); - if( image == NULL ) error("not enough memory."); - image->data = (float *) calloc( (size_t) (xsize*ysize), sizeof(float) ); - if( image->data == NULL ) error("not enough memory."); - - /* set image size */ - image->xsize = xsize; - image->ysize = ysize; - - return image; -} - -/*----------------------------------------------------------------------------*/ -/** Create a new image_double of size 'xsize' times 'ysize' - with the data pointed by 'data'. - */ -static image_double new_image_double_ptr( unsigned int xsize, - unsigned int ysize, float * data ) -{ - image_double image; - - /* check parameters */ - if( xsize == 0 || ysize == 0 ) - error("new_image_double_ptr: invalid image size."); - if( data == NULL ) error("new_image_double_ptr: NULL data pointer."); - - /* get memory */ - image = (image_double) malloc( sizeof(struct image_double_s) ); - if( image == NULL ) error("not enough memory."); - - /* set image */ - image->xsize = xsize; - image->ysize = ysize; - image->data = data; - - return image; -} - - -/*----------------------------------------------------------------------------*/ -/*----------------------------- Gaussian filter ------------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Compute a Gaussian kernel of length 'kernel->dim', - standard deviation 'sigma', and centered at value 'mean'. - - For example, if mean=0.5, the Gaussian will be centered - in the middle point between values 'kernel->values[0]' - and 'kernel->values[1]'. - */ -static void gaussian_kernel(ntuple_list kernel, float sigma, float mean) -{ - float sum = 0.0; - float val; - unsigned int i; - - /* check parameters */ - if( kernel == NULL || kernel->values == NULL ) - error("gaussian_kernel: invalid n-tuple 'kernel'."); - if( sigma <= 0.0 ) error("gaussian_kernel: 'sigma' must be positive."); - - /* compute Gaussian kernel */ - if( kernel->max_size < 1 ) enlarge_ntuple_list(kernel); - kernel->size = 1; - for(i=0;idim;i++) - { - val = ( (float) i - mean ) / sigma; - kernel->values[i] = exp( -0.5 * val * val ); - sum += kernel->values[i]; - } - - /* normalization */ - if( sum >= 0.0 ) for(i=0;idim;i++) kernel->values[i] /= sum; -} - -/*----------------------------------------------------------------------------*/ -/** Scale the input image 'in' by a factor 'scale' by Gaussian sub-sampling. - - For example, scale=0.8 will give a result at 80% of the original size. - - The image is convolved with a Gaussian kernel - @f[ - G(x,y) = \frac{1}{2\pi\sigma^2} e^{-\frac{x^2+y^2}{2\sigma^2}} - @f] - before the sub-sampling to prevent aliasing. - - The standard deviation sigma given by: - - sigma = sigma_scale / scale, if scale < 1.0 - - sigma = sigma_scale, if scale >= 1.0 - - To be able to sub-sample at non-integer steps, some interpolation - is needed. In this implementation, the interpolation is done by - the Gaussian kernel, so both operations (filtering and sampling) - are done at the same time. The Gaussian kernel is computed - centered on the coordinates of the required sample. In this way, - when applied, it gives directly the result of convolving the image - with the kernel and interpolated to that particular position. - - A fast algorithm is done using the separability of the Gaussian - kernel. Applying the 2D Gaussian kernel is equivalent to applying - first a horizontal 1D Gaussian kernel and then a vertical 1D - Gaussian kernel (or the other way round). The reason is that - @f[ - G(x,y) = G(x) * G(y) - @f] - where - @f[ - G(x) = \frac{1}{\sqrt{2\pi}\sigma} e^{-\frac{x^2}{2\sigma^2}}. - @f] - The algorithm first applies a combined Gaussian kernel and sampling - in the x axis, and then the combined Gaussian kernel and sampling - in the y axis. - */ -static image_double gaussian_sampler( image_double in, float scale, - float sigma_scale ) -{ - image_double aux,out; - ntuple_list kernel; - unsigned int N,M,h,n,x,y,i; - int xc,yc,j,double_x_size,double_y_size; - float sigma,xx,yy,sum,prec; - - /* check parameters */ - if( in == NULL || in->data == NULL || in->xsize == 0 || in->ysize == 0 ) - error("gaussian_sampler: invalid image."); - if( scale <= 0.0 ) error("gaussian_sampler: 'scale' must be positive."); - if( sigma_scale <= 0.0 ) - error("gaussian_sampler: 'sigma_scale' must be positive."); - - /* compute new image size and get memory for images */ - if( in->xsize * scale > (float) UINT_MAX || - in->ysize * scale > (float) UINT_MAX ) - error("gaussian_sampler: the output image size exceeds the handled size."); - N = (unsigned int) ceil( in->xsize * scale ); - M = (unsigned int) ceil( in->ysize * scale ); - aux = new_image_double(N,in->ysize); - out = new_image_double(N,M); - - /* sigma, kernel size and memory for the kernel */ - sigma = scale < 1.0 ? sigma_scale / scale : sigma_scale; - /* - The size of the kernel is selected to guarantee that the - the first discarded term is at least 10^prec times smaller - than the central value. For that, h should be larger than x, with - e^(-x^2/2sigma^2) = 1/10^prec. - Then, - x = sigma * sqrt( 2 * prec * ln(10) ). - */ - prec = 3.0; - h = (unsigned int) ceil( sigma * sqrt( 2.0 * prec * log(10.0) ) ); - n = 1+2*h; /* kernel size */ - kernel = new_ntuple_list(n); - - /* auxiliary double image size variables */ - double_x_size = (int) (2 * in->xsize); - double_y_size = (int) (2 * in->ysize); - - /* First subsampling: x axis */ - for(x=0;xxsize;x++) - { - /* - x is the coordinate in the new image. - xx is the corresponding x-value in the original size image. - xc is the integer value, the pixel coordinate of xx. - */ - xx = (float) x / scale; - /* coordinate (0.0,0.0) is in the center of pixel (0,0), - so the pixel with xc=0 get the values of xx from -0.5 to 0.5 */ - xc = (int) floor( xx + 0.5 ); - gaussian_kernel( kernel, sigma, (float) h + xx - (float) xc ); - /* the kernel must be computed for each x because the fine - offset xx-xc is different in each case */ - - for(y=0;yysize;y++) - { - sum = 0.0; - for(i=0;idim;i++) - { - j = xc - h + i; - - /* symmetry boundary condition */ - while( j < 0 ) j += double_x_size; - while( j >= double_x_size ) j -= double_x_size; - if( j >= (int) in->xsize ) j = double_x_size-1-j; - - sum += in->data[ j + y * in->xsize ] * kernel->values[i]; - } - aux->data[ x + y * aux->xsize ] = sum; - } - } - - /* Second subsampling: y axis */ - for(y=0;yysize;y++) - { - /* - y is the coordinate in the new image. - yy is the corresponding x-value in the original size image. - yc is the integer value, the pixel coordinate of xx. - */ - yy = (float) y / scale; - /* coordinate (0.0,0.0) is in the center of pixel (0,0), - so the pixel with yc=0 get the values of yy from -0.5 to 0.5 */ - yc = (int) floor( yy + 0.5 ); - gaussian_kernel( kernel, sigma, (float) h + yy - (float) yc ); - /* the kernel must be computed for each y because the fine - offset yy-yc is different in each case */ - - for(x=0;xxsize;x++) - { - sum = 0.0; - for(i=0;idim;i++) - { - j = yc - h + i; - - /* symmetry boundary condition */ - while( j < 0 ) j += double_y_size; - while( j >= double_y_size ) j -= double_y_size; - if( j >= (int) in->ysize ) j = double_y_size-1-j; - - sum += aux->data[ x + j * aux->xsize ] * kernel->values[i]; - } - out->data[ x + y * out->xsize ] = sum; - } - } - - /* free memory */ - free_ntuple_list(kernel); - free_image_double(aux); - - return out; -} - - -/*----------------------------------------------------------------------------*/ -/*--------------------------------- Gradient ---------------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Computes the direction of the level line of 'in' at each point. - - The result is: - - an image_double with the angle at each pixel, or NOTDEF if not defined. - - the image_double 'modgrad' (a pointer is passed as argument) - with the gradient magnitude at each point. - - a list of pixels 'list_p' roughly ordered by decreasing - gradient magnitude. (The order is made by classifying points - into bins by gradient magnitude. The parameters 'n_bins' and - 'max_grad' specify the number of bins and the gradient modulus - at the highest bin. The pixels in the list would be in - decreasing gradient magnitude, up to a precision of the size of - the bins.) - - a pointer 'mem_p' to the memory used by 'list_p' to be able to - free the memory when it is not used anymore. - */ -static image_double ll_angle( image_double in, float threshold, - struct coorlist ** list_p, void ** mem_p, - image_double * modgrad, unsigned int n_bins ) -{ - image_double g; - unsigned int n,p,x,y,adr,i; - float com1,com2,gx,gy,norm,norm2; - /* the rest of the variables are used for pseudo-ordering - the gradient magnitude values */ - int list_count = 0; - struct coorlist * list; - struct coorlist ** range_l_s; /* array of pointers to start of bin list */ - struct coorlist ** range_l_e; /* array of pointers to end of bin list */ - struct coorlist * start; - struct coorlist * end; - float max_grad = 0.0; - - /* check parameters */ - if( in == NULL || in->data == NULL || in->xsize == 0 || in->ysize == 0 ) - error("ll_angle: invalid image."); - if( threshold < 0.0 ) error("ll_angle: 'threshold' must be positive."); - if( list_p == NULL ) error("ll_angle: NULL pointer 'list_p'."); - if( mem_p == NULL ) error("ll_angle: NULL pointer 'mem_p'."); - if( modgrad == NULL ) error("ll_angle: NULL pointer 'modgrad'."); - if( n_bins == 0 ) error("ll_angle: 'n_bins' must be positive."); - - /* image size shortcuts */ - n = in->ysize; - p = in->xsize; - - /* allocate output image */ - g = new_image_double(in->xsize,in->ysize); - - /* get memory for the image of gradient modulus */ - *modgrad = new_image_double(in->xsize,in->ysize); - - /* get memory for "ordered" list of pixels */ - list = (struct coorlist *) calloc( (size_t) (n*p), sizeof(struct coorlist) ); - *mem_p = (void *) list; - range_l_s = (struct coorlist **) calloc( (size_t) n_bins, - sizeof(struct coorlist *) ); - range_l_e = (struct coorlist **) calloc( (size_t) n_bins, - sizeof(struct coorlist *) ); - if( list == NULL || range_l_s == NULL || range_l_e == NULL ) - error("not enough memory."); - for(i=0;idata[(n-1)*p+x] = NOTDEF; - for(y=0;ydata[p*y+p-1] = NOTDEF; - - /* compute gradient on the remaining pixels */ - for(x=0;xdata[adr]); - com1 = in->data[adr+p+1] - in->data[adr]; - com2 = in->data[adr+1] - in->data[adr+p]; - - gx = com1+com2; /* gradient x component */ - gy = com1-com2; /* gradient y component */ - norm2 = gx*gx+gy*gy; - norm = sqrt( norm2 / 4.0 ); /* gradient norm */ - - (*modgrad)->data[adr] = norm; /* store gradient norm */ - - if( norm <= threshold ) /* norm too small, gradient no defined */ - g->data[adr] = NOTDEF; /* gradient angle not defined */ - else - { - /* gradient angle computation */ - g->data[adr] = atan2(gx,-gy); - - /* look for the maximum of the gradient */ - if( norm > max_grad ) max_grad = norm; - } - } - - /* compute histogram of gradient values */ - for(x=0;xdata[y*p+x]; - - /* store the point in the right bin according to its norm */ - i = (unsigned int) (norm * (float) n_bins / max_grad); - if( i >= n_bins ) i = n_bins-1; - if( range_l_e[i] == NULL ) - range_l_s[i] = range_l_e[i] = list+list_count++; - else - { - range_l_e[i]->next = list+list_count; - range_l_e[i] = list+list_count++; - } - range_l_e[i]->x = (int) x; - range_l_e[i]->y = (int) y; - range_l_e[i]->next = NULL; - } - - /* Make the list of pixels (almost) ordered by norm value. - It starts by the larger bin, so the list starts by the - pixels with the highest gradient value. Pixels would be ordered - by norm value, up to a precision given by max_grad/n_bins. - */ - for(i=n_bins-1; i>0 && range_l_s[i]==NULL; i--); - start = range_l_s[i]; - end = range_l_e[i]; - if( start != NULL ) - while(i>0) - { - --i; - if( range_l_s[i] != NULL ) - { - end->next = range_l_s[i]; - end = range_l_e[i]; - } - } - *list_p = start; - - /* free memory */ - free( (void *) range_l_s ); - free( (void *) range_l_e ); - - return g; -} - -/*----------------------------------------------------------------------------*/ -/** Is point (x,y) aligned to angle theta, up to precision 'prec'? - */ -static int isaligned( int x, int y, image_double angles, float theta, - float prec ) -{ - float a; - - /* check parameters */ - if( angles == NULL || angles->data == NULL ) - error("isaligned: invalid image 'angles'."); - if( x < 0 || y < 0 || x >= (int) angles->xsize || y >= (int) angles->ysize ) - error("isaligned: (x,y) out of the image."); - if( prec < 0.0 ) error("isaligned: 'prec' must be positive."); - - /* angle at pixel (x,y) */ - a = angles->data[ x + y * angles->xsize ]; - - /* pixels whose level-line angle is not defined - are considered as NON-aligned */ - if( a == NOTDEF ) return FALSE; /* there is no need to call the function - 'double_equal' here because there is - no risk of problems related to the - comparison doubles, we are only - interested in the exact NOTDEF value */ - - /* it is assumed that 'theta' and 'a' are in the range [-pi,pi] */ - theta -= a; - if( theta < 0.0 ) theta = -theta; - if( theta > M_3_2_PI ) - { - theta -= M_2__PI; - if( theta < 0.0 ) theta = -theta; - } - - return theta <= prec; -} - -/*----------------------------------------------------------------------------*/ -/** Absolute value angle difference. - */ -static double angle_diff(float a, float b) -{ - a -= b; - while( a <= -M_PI ) a += M_2__PI; - while( a > M_PI ) a -= M_2__PI; - if( a < 0.0 ) a = -a; - return a; -} - -/*----------------------------------------------------------------------------*/ -/** Signed angle difference. - */ -static double angle_diff_signed(float a, float b) -{ - a -= b; - while( a <= -M_PI ) a += M_2__PI; - while( a > M_PI ) a -= M_2__PI; - return a; -} - - -/*----------------------------------------------------------------------------*/ -/*----------------------------- NFA computation ------------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Computes the natural logarithm of the absolute value of - the gamma function of x using the Lanczos approximation. - See http://www.rskey.org/gamma.htm - - The formula used is - @f[ - \Gamma(x) = \frac{ \sum_{n=0}^{N} q_n x^n }{ \Pi_{n=0}^{N} (x+n) } - (x+5.5)^{x+0.5} e^{-(x+5.5)} - @f] - so - @f[ - \log\Gamma(x) = \log\left( \sum_{n=0}^{N} q_n x^n \right) - + (x+0.5) \log(x+5.5) - (x+5.5) - \sum_{n=0}^{N} \log(x+n) - @f] - and - q0 = 75122.6331530, - q1 = 80916.6278952, - q2 = 36308.2951477, - q3 = 8687.24529705, - q4 = 1168.92649479, - q5 = 83.8676043424, - q6 = 2.50662827511. - */ -static double log_gamma_lanczos(float x) -{ - static float q[7] = { 75122.6331530, 80916.6278952, 36308.2951477, - 8687.24529705, 1168.92649479, 83.8676043424, - 2.50662827511 }; - float a = (x+0.5) * log(x+5.5) - (x+5.5); - float b = 0.0; - int n; - - for(n=0;n<7;n++) - { - a -= log( x + (float) n ); - b += q[n] * pow( x, (float) n ); - } - return a + log(b); -} - -/*----------------------------------------------------------------------------*/ -/** Computes the natural logarithm of the absolute value of - the gamma function of x using Windschitl method. - See http://www.rskey.org/gamma.htm - - The formula used is - @f[ - \Gamma(x) = \sqrt{\frac{2\pi}{x}} \left( \frac{x}{e} - \sqrt{ x\sinh(1/x) + \frac{1}{810x^6} } \right)^x - @f] - so - @f[ - \log\Gamma(x) = 0.5\log(2\pi) + (x-0.5)\log(x) - x - + 0.5x\log\left( x\sinh(1/x) + \frac{1}{810x^6} \right). - @f] - This formula is a good approximation when x > 15. - */ -static double log_gamma_windschitl(float x) -{ - return 0.918938533204673 + (x-0.5)*log(x) - x - + 0.5*x*log( x*sinh(1/x) + 1/(810.0*pow(x,6.0)) ); -} - -/*----------------------------------------------------------------------------*/ -/** Computes the natural logarithm of the absolute value of - the gamma function of x. When x>15 use log_gamma_windschitl(), - otherwise use log_gamma_lanczos(). - */ -#define log_gamma(x) ((x)>15.0?log_gamma_windschitl(x):log_gamma_lanczos(x)) - -/*----------------------------------------------------------------------------*/ -/** Size of the table to store already computed inverse values. - */ -#define TABSIZE 100000 - -/*----------------------------------------------------------------------------*/ -/** Computes -log10(NFA). - - NFA stands for Number of False Alarms: - @f[ - \mathrm{NFA} = NT \cdot B(n,k,p) - @f] - - - NT - number of tests - - B(n,k,p) - tail of binomial distribution with parameters n,k and p: - @f[ - B(n,k,p) = \sum_{j=k}^n - \left(\begin{array}{c}n\\j\end{array}\right) - p^{j} (1-p)^{n-j} - @f] - - The value -log10(NFA) is equivalent but more intuitive than NFA: - - -1 corresponds to 10 mean false alarms - - 0 corresponds to 1 mean false alarm - - 1 corresponds to 0.1 mean false alarms - - 2 corresponds to 0.01 mean false alarms - - ... - - Used this way, the bigger the value, better the detection, - and a logarithmic scale is used. - - @param n,k,p binomial parameters. - @param logNT logarithm of Number of Tests - - The computation is based in the gamma function by the following - relation: - @f[ - \left(\begin{array}{c}n\\k\end{array}\right) - = \frac{ \Gamma(n+1) }{ \Gamma(k+1) \cdot \Gamma(n-k+1) }. - @f] - We use efficient algorithms to compute the logarithm of - the gamma function. - - To make the computation faster, not all the sum is computed, part - of the terms are neglected based on a bound to the error obtained - (an error of 10% in the result is accepted). - */ -static double nfa(int n, int k, float p, float logNT) -{ - static float inv[TABSIZE]; /* table to keep computed inverse values */ - float tolerance = 0.1; /* an error of 10% in the result is accepted */ - float log1term,term,bin_term,mult_term,bin_tail,err,p_term; - int i; - - /* check parameters */ - if( n<0 || k<0 || k>n || p<=0.0 || p>=1.0 ) - error("nfa: wrong n, k or p values."); - - /* trivial cases */ - if( n==0 || k==0 ) return -logNT; - if( n==k ) return -logNT - (float) n * log10(p); - - /* probability term */ - p_term = p / (1.0-p); - - /* compute the first term of the series */ - /* - binomial_tail(n,k,p) = sum_{i=k}^n bincoef(n,i) * p^i * (1-p)^{n-i} - where bincoef(n,i) are the binomial coefficients. - But - bincoef(n,k) = gamma(n+1) / ( gamma(k+1) * gamma(n-k+1) ). - We use this to compute the first term. Actually the log of it. - */ - log1term = log_gamma( (float) n + 1.0 ) - log_gamma( (float) k + 1.0 ) - - log_gamma( (float) (n-k) + 1.0 ) - + (float) k * log(p) + (float) (n-k) * log(1.0-p); - term = exp(log1term); - - /* in some cases no more computations are needed */ - if( double_equal(term,0.0) ) /* the first term is almost zero */ - { - if( (float) k > (float) n * p ) /* at begin or end of the tail? */ - return -log1term / M_LN10 - logNT; /* end: use just the first term */ - else - return -logNT; /* begin: the tail is roughly 1 */ - } - - /* compute more terms if needed */ - bin_tail = term; - for(i=k+1;i<=n;i++) - { - /* - As - term_i = bincoef(n,i) * p^i * (1-p)^(n-i) - and - bincoef(n,i)/bincoef(n,i-1) = n-1+1 / i, - then, - term_i / term_i-1 = (n-i+1)/i * p/(1-p) - and - term_i = term_i-1 * (n-i+1)/i * p/(1-p). - 1/i is stored in a table as they are computed, - because divisions are expensive. - p/(1-p) is computed only once and stored in 'p_term'. - */ - bin_term = (float) (n-i+1) * ( ii. - Then, the error on the binomial tail when truncated at - the i term can be bounded by a geometric series of form - term_i * sum mult_term_i^j. */ - err = term * ( ( 1.0 - pow( mult_term, (float) (n-i+1) ) ) / - (1.0-mult_term) - 1.0 ); - - /* One wants an error at most of tolerance*final_result, or: - tolerance * abs(-log10(bin_tail)-logNT). - Now, the error that can be accepted on bin_tail is - given by tolerance*final_result divided by the derivative - of -log10(x) when x=bin_tail. that is: - tolerance * abs(-log10(bin_tail)-logNT) / (1/bin_tail) - Finally, we truncate the tail if the error is less than: - tolerance * abs(-log10(bin_tail)-logNT) * bin_tail */ - if( err < tolerance * fabs(-log10(bin_tail)-logNT) * bin_tail ) break; - } - } - return -log10(bin_tail) - logNT; -} - - -/*----------------------------------------------------------------------------*/ -/*--------------------------- Rectangle structure ----------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Rectangle structure: line segment with width. - */ -struct rect -{ - float x1,y1,x2,y2; /* first and second point of the line segment */ - float width; /* rectangle width */ - float x,y; /* center of the rectangle */ - float theta; /* angle */ - float dx,dy; /* (dx,dy) is vector oriented as the line segment */ - float prec; /* tolerance angle */ - float p; /* probability of a point with angle within 'prec' */ -}; - -/*----------------------------------------------------------------------------*/ -/** Copy one rectangle structure to another. - */ -static void rect_copy(struct rect * in, struct rect * out) -{ - /* check parameters */ - if( in == NULL || out == NULL ) error("rect_copy: invalid 'in' or 'out'."); - - /* copy values */ - out->x1 = in->x1; - out->y1 = in->y1; - out->x2 = in->x2; - out->y2 = in->y2; - out->width = in->width; - out->x = in->x; - out->y = in->y; - out->theta = in->theta; - out->dx = in->dx; - out->dy = in->dy; - out->prec = in->prec; - out->p = in->p; -} - -/*----------------------------------------------------------------------------*/ -/** Rectangle points iterator. - - The integer coordinates of pixels inside a rectangle are - iteratively explored. This structure keep track of the process and - functions ri_ini(), ri_inc(), ri_end(), and ri_del() are used in - the process. An example of how to use the iterator is as follows: - \code - - struct rect * rec = XXX; // some rectangle - rect_iter * i; - for( i=ri_ini(rec); !ri_end(i); ri_inc(i) ) - { - // your code, using 'i->x' and 'i->y' as coordinates - } - ri_del(i); // delete iterator - - \endcode - The pixels are explored 'column' by 'column', where we call - 'column' a set of pixels with the same x value that are inside the - rectangle. The following is an schematic representation of a - rectangle, the 'column' being explored is marked by colons, and - the current pixel being explored is 'x,y'. - \verbatim - - vx[1],vy[1] - * * - * * - * * - * ye - * : * - vx[0],vy[0] : * - * : * - * x,y * - * : * - * : vx[2],vy[2] - * : * - y ys * - ^ * * - | * * - | * * - +---> x vx[3],vy[3] - - \endverbatim - The first 'column' to be explored is the one with the smaller x - value. Each 'column' is explored starting from the pixel of the - 'column' (inside the rectangle) with the smallest y value. - - The four corners of the rectangle are stored in order that rotates - around the corners at the arrays 'vx[]' and 'vy[]'. The first - point is always the one with smaller x value. - - 'x' and 'y' are the coordinates of the pixel being explored. 'ys' - and 'ye' are the start and end values of the current column being - explored. So, 'ys' < 'ye'. - */ -typedef struct -{ - float vx[4]; /* rectangle's corner X coordinates in circular order */ - float vy[4]; /* rectangle's corner Y coordinates in circular order */ - float ys,ye; /* start and end Y values of current 'column' */ - int x,y; /* coordinates of currently explored pixel */ -} rect_iter; - -/*----------------------------------------------------------------------------*/ -/** Interpolate y value corresponding to 'x' value given, in - the line 'x1,y1' to 'x2,y2'; if 'x1=x2' return the smaller - of 'y1' and 'y2'. - - The following restrictions are required: - - x1 <= x2 - - x1 <= x - - x <= x2 - */ -static double inter_low(float x, float x1, float y1, float x2, float y2) -{ - /* check parameters */ - if( x1 > x2 || x < x1 || x > x2 ) - error("inter_low: unsuitable input, 'x1>x2' or 'xx2'."); - - /* interpolation */ - if( double_equal(x1,x2) && y1y2 ) return y2; - return y1 + (x-x1) * (y2-y1) / (x2-x1); -} - -/*----------------------------------------------------------------------------*/ -/** Interpolate y value corresponding to 'x' value given, in - the line 'x1,y1' to 'x2,y2'; if 'x1=x2' return the larger - of 'y1' and 'y2'. - - The following restrictions are required: - - x1 <= x2 - - x1 <= x - - x <= x2 - */ -static double inter_hi(float x, float x1, float y1, float x2, float y2) -{ - /* check parameters */ - if( x1 > x2 || x < x1 || x > x2 ) - error("inter_hi: unsuitable input, 'x1>x2' or 'xx2'."); - - /* interpolation */ - if( double_equal(x1,x2) && y1y2 ) return y1; - return y1 + (x-x1) * (y2-y1) / (x2-x1); -} - -/*----------------------------------------------------------------------------*/ -/** Free memory used by a rectangle iterator. - */ -static void ri_del(rect_iter * iter) -{ - if( iter == NULL ) error("ri_del: NULL iterator."); - free( (void *) iter ); -} - -/*----------------------------------------------------------------------------*/ -/** Check if the iterator finished the full iteration. - - See details in \ref rect_iter - */ -static int ri_end(rect_iter * i) -{ - /* check input */ - if( i == NULL ) error("ri_end: NULL iterator."); - - /* if the current x value is larger than the largest - x value in the rectangle (vx[2]), we know the full - exploration of the rectangle is finished. */ - return (float)(i->x) > i->vx[2]; -} - -/*----------------------------------------------------------------------------*/ -/** Increment a rectangle iterator. - - See details in \ref rect_iter - */ -static void ri_inc(rect_iter * i) -{ - /* check input */ - if( i == NULL ) error("ri_inc: NULL iterator."); - - /* if not at end of exploration, - increase y value for next pixel in the 'column' */ - if( !ri_end(i) ) i->y++; - - /* if the end of the current 'column' is reached, - and it is not the end of exploration, - advance to the next 'column' */ - while( (float) (i->y) > i->ye && !ri_end(i) ) - { - /* increase x, next 'column' */ - i->x++; - - /* if end of exploration, return */ - if( ri_end(i) ) return; - - /* update lower y limit (start) for the new 'column'. - - We need to interpolate the y value that corresponds to the - lower side of the rectangle. The first thing is to decide if - the corresponding side is - - vx[0],vy[0] to vx[3],vy[3] or - vx[3],vy[3] to vx[2],vy[2] - - Then, the side is interpolated for the x value of the - 'column'. But, if the side is vertical (as it could happen if - the rectangle is vertical and we are dealing with the first - or last 'columns') then we pick the lower value of the side - by using 'inter_low'. - */ - if( (float) i->x < i->vx[3] ) - i->ys = inter_low((float)i->x,i->vx[0],i->vy[0],i->vx[3],i->vy[3]); - else - i->ys = inter_low((float)i->x,i->vx[3],i->vy[3],i->vx[2],i->vy[2]); - - /* update upper y limit (end) for the new 'column'. - - We need to interpolate the y value that corresponds to the - upper side of the rectangle. The first thing is to decide if - the corresponding side is - - vx[0],vy[0] to vx[1],vy[1] or - vx[1],vy[1] to vx[2],vy[2] - - Then, the side is interpolated for the x value of the - 'column'. But, if the side is vertical (as it could happen if - the rectangle is vertical and we are dealing with the first - or last 'columns') then we pick the lower value of the side - by using 'inter_low'. - */ - if( (float)i->x < i->vx[1] ) - i->ye = inter_hi((float)i->x,i->vx[0],i->vy[0],i->vx[1],i->vy[1]); - else - i->ye = inter_hi((float)i->x,i->vx[1],i->vy[1],i->vx[2],i->vy[2]); - - /* new y */ - i->y = (int) ceil(i->ys); - } -} - -/*----------------------------------------------------------------------------*/ -/** Create and initialize a rectangle iterator. - - See details in \ref rect_iter - */ -static rect_iter * ri_ini(struct rect * r) -{ - float vx[4],vy[4]; - int n,offset; - rect_iter * i; - - /* check parameters */ - if( r == NULL ) error("ri_ini: invalid rectangle."); - - /* get memory */ - i = (rect_iter *) malloc(sizeof(rect_iter)); - if( i == NULL ) error("ri_ini: Not enough memory."); - - /* build list of rectangle corners ordered - in a circular way around the rectangle */ - vx[0] = r->x1 - r->dy * r->width / 2.0; - vy[0] = r->y1 + r->dx * r->width / 2.0; - vx[1] = r->x2 - r->dy * r->width / 2.0; - vy[1] = r->y2 + r->dx * r->width / 2.0; - vx[2] = r->x2 + r->dy * r->width / 2.0; - vy[2] = r->y2 - r->dx * r->width / 2.0; - vx[3] = r->x1 + r->dy * r->width / 2.0; - vy[3] = r->y1 - r->dx * r->width / 2.0; - - /* compute rotation of index of corners needed so that the first - point has the smaller x. - - if one side is vertical, thus two corners have the same smaller x - value, the one with the largest y value is selected as the first. - */ - if( r->x1 < r->x2 && r->y1 <= r->y2 ) offset = 0; - else if( r->x1 >= r->x2 && r->y1 < r->y2 ) offset = 1; - else if( r->x1 > r->x2 && r->y1 >= r->y2 ) offset = 2; - else offset = 3; - - /* apply rotation of index. */ - for(n=0; n<4; n++) - { - i->vx[n] = vx[(offset+n)%4]; - i->vy[n] = vy[(offset+n)%4]; - } - - /* Set an initial condition. - - The values are set to values that will cause 'ri_inc' (that will - be called immediately) to initialize correctly the first 'column' - and compute the limits 'ys' and 'ye'. - - 'y' is set to the integer value of vy[0], the starting corner. - - 'ys' and 'ye' are set to very small values, so 'ri_inc' will - notice that it needs to start a new 'column'. - - The smallest integer coordinate inside of the rectangle is - 'ceil(vx[0])'. The current 'x' value is set to that value minus - one, so 'ri_inc' (that will increase x by one) will advance to - the first 'column'. - */ - i->x = (int) ceil(i->vx[0]) - 1; - i->y = (int) ceil(i->vy[0]); - i->ys = i->ye = -DBL_MAX; - - /* advance to the first pixel */ - ri_inc(i); - - return i; -} - -/*----------------------------------------------------------------------------*/ -/** Compute a rectangle's NFA value. - */ -static float rect_nfa(struct rect * rec, image_double angles, float logNT) -{ - rect_iter * i; - int pts = 0; - int alg = 0; - - /* check parameters */ - if( rec == NULL ) error("rect_nfa: invalid rectangle."); - if( angles == NULL ) error("rect_nfa: invalid 'angles'."); - - /* compute the total number of pixels and of aligned points in 'rec' */ - for(i=ri_ini(rec); !ri_end(i); ri_inc(i)) /* rectangle iterator */ - if( i->x >= 0 && i->y >= 0 && - i->x < (int) angles->xsize && i->y < (int) angles->ysize ) - { - ++pts; /* total number of pixels counter */ - if( isaligned(i->x, i->y, angles, rec->theta, rec->prec) ) - ++alg; /* aligned points counter */ - } - ri_del(i); /* delete iterator */ - - return nfa(pts,alg,rec->p,logNT); /* compute NFA value */ -} - - -/*----------------------------------------------------------------------------*/ -/*---------------------------------- Regions ---------------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** Compute region's angle as the principal inertia axis of the region. - - The following is the region inertia matrix A: - @f[ - - A = \left(\begin{array}{cc} - Ixx & Ixy \\ - Ixy & Iyy \\ - \end{array}\right) - - @f] - where - - Ixx = sum_i G(i).(y_i - cx)^2 - - Iyy = sum_i G(i).(x_i - cy)^2 - - Ixy = - sum_i G(i).(x_i - cx).(y_i - cy) - - and - - G(i) is the gradient norm at pixel i, used as pixel's weight. - - x_i and y_i are the coordinates of pixel i. - - cx and cy are the coordinates of the center of th region. - - lambda1 and lambda2 are the eigenvalues of matrix A, - with lambda1 >= lambda2. They are found by solving the - characteristic polynomial: - - det( lambda I - A) = 0 - - that gives: - - lambda1 = ( Ixx + Iyy + sqrt( (Ixx-Iyy)^2 + 4.0*Ixy*Ixy) ) / 2 - - lambda2 = ( Ixx + Iyy - sqrt( (Ixx-Iyy)^2 + 4.0*Ixy*Ixy) ) / 2 - - To get the line segment direction we want to get the angle the - eigenvector associated to the smallest eigenvalue. We have - to solve for a,b in: - - a.Ixx + b.Ixy = a.lambda2 - - a.Ixy + b.Iyy = b.lambda2 - - We want the angle theta = atan(b/a). It can be computed with - any of the two equations: - - theta = atan( (lambda2-Ixx) / Ixy ) - - or - - theta = atan( Ixy / (lambda2-Iyy) ) - - When |Ixx| > |Iyy| we use the first, otherwise the second (just to - get better numeric precision). - */ -static float get_theta( struct point * reg, int reg_size, float x, float y, - image_double modgrad, float reg_angle, float prec ) -{ - float lambda,theta,weight; - float Ixx = 0.0; - float Iyy = 0.0; - float Ixy = 0.0; - int i; - - /* check parameters */ - if( reg == NULL ) error("get_theta: invalid region."); - if( reg_size <= 1 ) error("get_theta: region size <= 1."); - if( modgrad == NULL || modgrad->data == NULL ) - error("get_theta: invalid 'modgrad'."); - if( prec < 0.0 ) error("get_theta: 'prec' must be positive."); - - /* compute inertia matrix */ - for(i=0; idata[ reg[i].x + reg[i].y * modgrad->xsize ]; - Ixx += ( (float) reg[i].y - y ) * ( (float) reg[i].y - y ) * weight; - Iyy += ( (float) reg[i].x - x ) * ( (float) reg[i].x - x ) * weight; - Ixy -= ( (float) reg[i].x - x ) * ( (float) reg[i].y - y ) * weight; - } - if( double_equal(Ixx,0.0) && double_equal(Iyy,0.0) && double_equal(Ixy,0.0) ) - error("get_theta: null inertia matrix."); - - /* compute smallest eigenvalue */ - lambda = 0.5 * ( Ixx + Iyy - sqrt( (Ixx-Iyy)*(Ixx-Iyy) + 4.0*Ixy*Ixy ) ); - - /* compute angle */ - theta = fabs(Ixx)>fabs(Iyy) ? atan2(lambda-Ixx,Ixy) : atan2(Ixy,lambda-Iyy); - - /* The previous procedure doesn't cares about orientation, - so it could be wrong by 180 degrees. Here is corrected if necessary. */ - if( angle_diff(theta,reg_angle) > prec ) theta += M_PI; - - return theta; -} - -/*----------------------------------------------------------------------------*/ -/** Computes a rectangle that covers a region of points. - */ -static void region2rect( struct point * reg, int reg_size, - image_double modgrad, float reg_angle, - float prec, float p, struct rect * rec ) -{ - float x,y,dx,dy,l,w,theta,weight,sum,l_min,l_max,w_min,w_max; - int i; - - /* check parameters */ - if( reg == NULL ) error("region2rect: invalid region."); - if( reg_size <= 1 ) error("region2rect: region size <= 1."); - if( modgrad == NULL || modgrad->data == NULL ) - error("region2rect: invalid image 'modgrad'."); - if( rec == NULL ) error("region2rect: invalid 'rec'."); - - /* center of the region: - - It is computed as the weighted sum of the coordinates - of all the pixels in the region. The norm of the gradient - is used as the weight of a pixel. The sum is as follows: - cx = \sum_i G(i).x_i - cy = \sum_i G(i).y_i - where G(i) is the norm of the gradient of pixel i - and x_i,y_i are its coordinates. - */ - x = y = sum = 0.0; - for(i=0; idata[ reg[i].x + reg[i].y * modgrad->xsize ]; - x += (float) reg[i].x * weight; - y += (float) reg[i].y * weight; - sum += weight; - } - if( sum <= 0.0 ) error("region2rect: weights sum equal to zero."); - x /= sum; - y /= sum; - - /* theta */ - theta = get_theta(reg,reg_size,x,y,modgrad,reg_angle,prec); - - /* length and width: - - 'l' and 'w' are computed as the distance from the center of the - region to pixel i, projected along the rectangle axis (dx,dy) and - to the orthogonal axis (-dy,dx), respectively. - - The length of the rectangle goes from l_min to l_max, where l_min - and l_max are the minimum and maximum values of l in the region. - Analogously, the width is selected from w_min to w_max, where - w_min and w_max are the minimum and maximum of w for the pixels - in the region. - */ - dx = cos(theta); - dy = sin(theta); - l_min = l_max = w_min = w_max = 0.0; - for(i=0; i l_max ) l_max = l; - if( l < l_min ) l_min = l; - if( w > w_max ) w_max = w; - if( w < w_min ) w_min = w; - } - - /* store values */ - rec->x1 = x + l_min * dx; - rec->y1 = y + l_min * dy; - rec->x2 = x + l_max * dx; - rec->y2 = y + l_max * dy; - rec->width = w_max - w_min; - rec->x = x; - rec->y = y; - rec->theta = theta; - rec->dx = dx; - rec->dy = dy; - rec->prec = prec; - rec->p = p; - - /* we impose a minimal width of one pixel - - A sharp horizontal or vertical step would produce a perfectly - horizontal or vertical region. The width computed would be - zero. But that corresponds to a one pixels width transition in - the image. - */ - if( rec->width < 1.0 ) rec->width = 1.0; -} - -/*----------------------------------------------------------------------------*/ -/** Build a region of pixels that share the same angle, up to a - tolerance 'prec', starting at point (x,y). - */ -static void region_grow( int x, int y, image_double angles, struct point * reg, - int * reg_size, float * reg_angle, image_char used, - float prec ) -{ - float sumdx,sumdy; - int xx,yy,i; - - /* check parameters */ - if( x < 0 || y < 0 || x >= (int) angles->xsize || y >= (int) angles->ysize ) - error("region_grow: (x,y) out of the image."); - if( angles == NULL || angles->data == NULL ) - error("region_grow: invalid image 'angles'."); - if( reg == NULL ) error("region_grow: invalid 'reg'."); - if( reg_size == NULL ) error("region_grow: invalid pointer 'reg_size'."); - if( reg_angle == NULL ) error("region_grow: invalid pointer 'reg_angle'."); - if( used == NULL || used->data == NULL ) - error("region_grow: invalid image 'used'."); - - /* first point of the region */ - *reg_size = 1; - reg[0].x = x; - reg[0].y = y; - *reg_angle = angles->data[x+y*angles->xsize]; /* region's angle */ - sumdx = cos(*reg_angle); - sumdy = sin(*reg_angle); - used->data[x+y*used->xsize] = USED; - - /* try neighbors as new region points */ - for(i=0; i<*reg_size; i++) - for(xx=reg[i].x-1; xx<=reg[i].x+1; xx++) - for(yy=reg[i].y-1; yy<=reg[i].y+1; yy++) - if( xx>=0 && yy>=0 && xx<(int)used->xsize && yy<(int)used->ysize && - used->data[xx+yy*used->xsize] != USED && - isaligned(xx,yy,angles,*reg_angle,prec) ) - { - /* add point */ - used->data[xx+yy*used->xsize] = USED; - reg[*reg_size].x = xx; - reg[*reg_size].y = yy; - ++(*reg_size); - - /* update region's angle */ - sumdx += cos( angles->data[xx+yy*angles->xsize] ); - sumdy += sin( angles->data[xx+yy*angles->xsize] ); - *reg_angle = atan2(sumdy,sumdx); - } -} - -/*----------------------------------------------------------------------------*/ -/** Try some rectangles variations to improve NFA value. Only if the - rectangle is not meaningful (i.e., log_nfa <= log_eps). - */ -static float rect_improve( struct rect * rec, image_double angles, - float logNT, float log_eps ) -{ - struct rect r; - float log_nfa,log_nfa_new; - float delta = 0.5; - float delta_2 = delta / 2.0; - int n; - - log_nfa = rect_nfa(rec,angles,logNT); - - if( log_nfa > log_eps ) return log_nfa; - - /* try finer precisions */ - rect_copy(rec,&r); - for(n=0; n<5; n++) - { - r.p /= 2.0; - r.prec = r.p * M_PI; - log_nfa_new = rect_nfa(&r,angles,logNT); - if( log_nfa_new > log_nfa ) - { - log_nfa = log_nfa_new; - rect_copy(&r,rec); - } - } - - if( log_nfa > log_eps ) return log_nfa; - - /* try to reduce width */ - rect_copy(rec,&r); - for(n=0; n<5; n++) - { - if( (r.width - delta) >= 0.5 ) - { - r.width -= delta; - log_nfa_new = rect_nfa(&r,angles,logNT); - if( log_nfa_new > log_nfa ) - { - rect_copy(&r,rec); - log_nfa = log_nfa_new; - } - } - } - - if( log_nfa > log_eps ) return log_nfa; - - /* try to reduce one side of the rectangle */ - rect_copy(rec,&r); - for(n=0; n<5; n++) - { - if( (r.width - delta) >= 0.5 ) - { - r.x1 += -r.dy * delta_2; - r.y1 += r.dx * delta_2; - r.x2 += -r.dy * delta_2; - r.y2 += r.dx * delta_2; - r.width -= delta; - log_nfa_new = rect_nfa(&r,angles,logNT); - if( log_nfa_new > log_nfa ) - { - rect_copy(&r,rec); - log_nfa = log_nfa_new; - } - } - } - - if( log_nfa > log_eps ) return log_nfa; - - /* try to reduce the other side of the rectangle */ - rect_copy(rec,&r); - for(n=0; n<5; n++) - { - if( (r.width - delta) >= 0.5 ) - { - r.x1 -= -r.dy * delta_2; - r.y1 -= r.dx * delta_2; - r.x2 -= -r.dy * delta_2; - r.y2 -= r.dx * delta_2; - r.width -= delta; - log_nfa_new = rect_nfa(&r,angles,logNT); - if( log_nfa_new > log_nfa ) - { - rect_copy(&r,rec); - log_nfa = log_nfa_new; - } - } - } - - if( log_nfa > log_eps ) return log_nfa; - - /* try even finer precisions */ - rect_copy(rec,&r); - for(n=0; n<5; n++) - { - r.p /= 2.0; - r.prec = r.p * M_PI; - log_nfa_new = rect_nfa(&r,angles,logNT); - if( log_nfa_new > log_nfa ) - { - log_nfa = log_nfa_new; - rect_copy(&r,rec); - } - } - - return log_nfa; -} - -/*----------------------------------------------------------------------------*/ -/** Reduce the region size, by elimination the points far from the - starting point, until that leads to rectangle with the right - density of region points or to discard the region if too small. - */ -static int reduce_region_radius( struct point * reg, int * reg_size, - image_double modgrad, float reg_angle, - float prec, float p, struct rect * rec, - image_char used, image_double angles, - float density_th ) -{ - float density,rad1,rad2,rad,xc,yc; - int i; - - /* check parameters */ - if( reg == NULL ) error("reduce_region_radius: invalid pointer 'reg'."); - if( reg_size == NULL ) - error("reduce_region_radius: invalid pointer 'reg_size'."); - if( prec < 0.0 ) error("reduce_region_radius: 'prec' must be positive."); - if( rec == NULL ) error("reduce_region_radius: invalid pointer 'rec'."); - if( used == NULL || used->data == NULL ) - error("reduce_region_radius: invalid image 'used'."); - if( angles == NULL || angles->data == NULL ) - error("reduce_region_radius: invalid image 'angles'."); - - /* compute region points density */ - density = (float) *reg_size / - ( dist(rec->x1,rec->y1,rec->x2,rec->y2) * rec->width ); - - /* if the density criterion is satisfied there is nothing to do */ - if( density >= density_th ) return TRUE; - - /* compute region's radius */ - xc = (float) reg[0].x; - yc = (float) reg[0].y; - rad1 = dist( xc, yc, rec->x1, rec->y1 ); - rad2 = dist( xc, yc, rec->x2, rec->y2 ); - rad = rad1 > rad2 ? rad1 : rad2; - - /* while the density criterion is not satisfied, remove farther pixels */ - while( density < density_th ) - { - rad *= 0.75; /* reduce region's radius to 75% of its value */ - - /* remove points from the region and update 'used' map */ - for(i=0; i<*reg_size; i++) - if( dist( xc, yc, (float) reg[i].x, (float) reg[i].y ) > rad ) - { - /* point not kept, mark it as NOTUSED */ - used->data[ reg[i].x + reg[i].y * used->xsize ] = NOTUSED; - /* remove point from the region */ - reg[i].x = reg[*reg_size-1].x; /* if i==*reg_size-1 copy itself */ - reg[i].y = reg[*reg_size-1].y; - --(*reg_size); - --i; /* to avoid skipping one point */ - } - - /* reject if the region is too small. - 2 is the minimal region size for 'region2rect' to work. */ - if( *reg_size < 2 ) return FALSE; - - /* re-compute rectangle */ - region2rect(reg,*reg_size,modgrad,reg_angle,prec,p,rec); - - /* re-compute region points density */ - density = (float) *reg_size / - ( dist(rec->x1,rec->y1,rec->x2,rec->y2) * rec->width ); - } - - /* if this point is reached, the density criterion is satisfied */ - return TRUE; -} - -/*----------------------------------------------------------------------------*/ -/** Refine a rectangle. - - For that, an estimation of the angle tolerance is performed by the - standard deviation of the angle at points near the region's - starting point. Then, a new region is grown starting from the same - point, but using the estimated angle tolerance. If this fails to - produce a rectangle with the right density of region points, - 'reduce_region_radius' is called to try to satisfy this condition. - */ -static int refine( struct point * reg, int * reg_size, image_double modgrad, - float reg_angle, float prec, float p, struct rect * rec, - image_char used, image_double angles, float density_th ) -{ - float angle,ang_d,mean_angle,tau,density,xc,yc,ang_c,sum,s_sum; - int i,n; - - /* check parameters */ - if( reg == NULL ) error("refine: invalid pointer 'reg'."); - if( reg_size == NULL ) error("refine: invalid pointer 'reg_size'."); - if( prec < 0.0 ) error("refine: 'prec' must be positive."); - if( rec == NULL ) error("refine: invalid pointer 'rec'."); - if( used == NULL || used->data == NULL ) - error("refine: invalid image 'used'."); - if( angles == NULL || angles->data == NULL ) - error("refine: invalid image 'angles'."); - - /* compute region points density */ - density = (float) *reg_size / - ( dist(rec->x1,rec->y1,rec->x2,rec->y2) * rec->width ); - - /* if the density criterion is satisfied there is nothing to do */ - if( density >= density_th ) return TRUE; - - /*------ First try: reduce angle tolerance ------*/ - - /* compute the new mean angle and tolerance */ - xc = (float) reg[0].x; - yc = (float) reg[0].y; - ang_c = angles->data[ reg[0].x + reg[0].y * angles->xsize ]; - sum = s_sum = 0.0; - n = 0; - for(i=0; i<*reg_size; i++) - { - used->data[ reg[i].x + reg[i].y * used->xsize ] = NOTUSED; - if( dist( xc, yc, (float) reg[i].x, (float) reg[i].y ) < rec->width ) - { - angle = angles->data[ reg[i].x + reg[i].y * angles->xsize ]; - ang_d = angle_diff_signed(angle,ang_c); - sum += ang_d; - s_sum += ang_d * ang_d; - ++n; - } - } - mean_angle = sum / (float) n; - tau = 2.0 * sqrt( (s_sum - 2.0 * mean_angle * sum) / (float) n - + mean_angle*mean_angle ); /* 2 * standard deviation */ - - /* find a new region from the same starting point and new angle tolerance */ - region_grow(reg[0].x,reg[0].y,angles,reg,reg_size,®_angle,used,tau); - - /* if the region is too small, reject */ - if( *reg_size < 2 ) return FALSE; - - /* re-compute rectangle */ - region2rect(reg,*reg_size,modgrad,reg_angle,prec,p,rec); - - /* re-compute region points density */ - density = (float) *reg_size / - ( dist(rec->x1,rec->y1,rec->x2,rec->y2) * rec->width ); - - /*------ Second try: reduce region radius ------*/ - if( density < density_th ) - return reduce_region_radius( reg, reg_size, modgrad, reg_angle, prec, p, - rec, used, angles, density_th ); - - /* if this point is reached, the density criterion is satisfied */ - return TRUE; -} - - -/*----------------------------------------------------------------------------*/ -/*-------------------------- Line Segment Detector ---------------------------*/ -/*----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** LSD full interface. - */ -float * LineSegmentDetection( int * n_out, - float * img, int X, int Y, - float scale, float sigma_scale, float quant, - float ang_th, float log_eps, float density_th, - int n_bins, - int ** reg_img, int * reg_x, int * reg_y ) -{ - image_double image; - ntuple_list out = new_ntuple_list(7); - float * return_value; - image_double scaled_image,angles,modgrad; - image_char used; - image_int region = NULL; - struct coorlist * list_p; - void * mem_p; - struct rect rec; - struct point * reg; - int reg_size,min_reg_size,i; - unsigned int xsize,ysize; - float rho,reg_angle,prec,p,log_nfa,logNT; - int ls_count = 0; /* line segments are numbered 1,2,3,... */ - - - /* check parameters */ - if( img == NULL || X <= 0 || Y <= 0 ) error("invalid image input."); - if( scale <= 0.0 ) error("'scale' value must be positive."); - if( sigma_scale <= 0.0 ) error("'sigma_scale' value must be positive."); - if( quant < 0.0 ) error("'quant' value must be positive."); - if( ang_th <= 0.0 || ang_th >= 180.0 ) - error("'ang_th' value must be in the range (0,180)."); - if( density_th < 0.0 || density_th > 1.0 ) - error("'density_th' value must be in the range [0,1]."); - if( n_bins <= 0 ) error("'n_bins' value must be positive."); - - - /* angle tolerance */ - prec = M_PI * ang_th / 180.0; - p = ang_th / 180.0; - rho = quant / sin(prec); /* gradient magnitude threshold */ - - - /* load and scale image (if necessary) and compute angle at each pixel */ - image = new_image_double_ptr( (unsigned int) X, (unsigned int) Y, img ); - if( scale != 1.0 ) - { - scaled_image = gaussian_sampler( image, scale, sigma_scale ); - angles = ll_angle( scaled_image, rho, &list_p, &mem_p, - &modgrad, (unsigned int) n_bins ); - free_image_double(scaled_image); - } - else - angles = ll_angle( image, rho, &list_p, &mem_p, &modgrad, - (unsigned int) n_bins ); - xsize = angles->xsize; - ysize = angles->ysize; - - /* Number of Tests - NT - - The theoretical number of tests is Np.(XY)^(5/2) - where X and Y are number of columns and rows of the image. - Np corresponds to the number of angle precisions considered. - As the procedure 'rect_improve' tests 5 times to halve the - angle precision, and 5 more times after improving other factors, - 11 different precision values are potentially tested. Thus, - the number of tests is - 11 * (X*Y)^(5/2) - whose logarithm value is - log10(11) + 5/2 * (log10(X) + log10(Y)). - */ - logNT = 5.0 * ( log10( (float) xsize ) + log10( (float) ysize ) ) / 2.0 - + log10(11.0); - min_reg_size = (int) (-logNT/log10(p)); /* minimal number of points in region - that can give a meaningful event */ - - - /* initialize some structures */ - if( reg_img != NULL && reg_x != NULL && reg_y != NULL ) /* save region data */ - region = new_image_int_ini(angles->xsize,angles->ysize,0); - used = new_image_char_ini(xsize,ysize,NOTUSED); - reg = (struct point *) calloc( (size_t) (xsize*ysize), sizeof(struct point) ); - if( reg == NULL ) error("not enough memory!"); - - - /* search for line segments */ - for(; list_p != NULL; list_p = list_p->next ) - if( used->data[ list_p->x + list_p->y * used->xsize ] == NOTUSED && - angles->data[ list_p->x + list_p->y * angles->xsize ] != NOTDEF ) - /* there is no risk of double comparison problems here - because we are only interested in the exact NOTDEF value */ - { - /* find the region of connected point and ~equal angle */ - region_grow( list_p->x, list_p->y, angles, reg, ®_size, - ®_angle, used, prec ); - - /* reject small regions */ - if( reg_size < min_reg_size ) continue; - - /* construct rectangular approximation for the region */ - region2rect(reg,reg_size,modgrad,reg_angle,prec,p,&rec); - - /* Check if the rectangle exceeds the minimal density of - region points. If not, try to improve the region. - The rectangle will be rejected if the final one does - not fulfill the minimal density condition. - This is an addition to the original LSD algorithm published in - "LSD: A Fast Line Segment Detector with a False Detection Control" - by R. Grompone von Gioi, J. Jakubowicz, J.M. Morel, and G. Randall. - The original algorithm is obtained with density_th = 0.0. - */ - if( !refine( reg, ®_size, modgrad, reg_angle, - prec, p, &rec, used, angles, density_th ) ) continue; - - /* compute NFA value */ - log_nfa = rect_improve(&rec,angles,logNT,log_eps); - if( log_nfa <= log_eps ) continue; - - /* A New Line Segment was found! */ - ++ls_count; /* increase line segment counter */ - - /* - The gradient was computed with a 2x2 mask, its value corresponds to - points with an offset of (0.5,0.5), that should be added to output. - The coordinates origin is at the center of pixel (0,0). - */ - rec.x1 += 0.5; rec.y1 += 0.5; - rec.x2 += 0.5; rec.y2 += 0.5; - - /* scale the result values if a subsampling was performed */ - if( scale != 1.0 ) - { - rec.x1 /= scale; rec.y1 /= scale; - rec.x2 /= scale; rec.y2 /= scale; - rec.width /= scale; - } - - /* add line segment found to output */ - add_7tuple( out, rec.x1, rec.y1, rec.x2, rec.y2, - rec.width, rec.p, log_nfa ); - - /* add region number to 'region' image if needed */ - if( region != NULL ) - for(i=0; idata[ reg[i].x + reg[i].y * region->xsize ] = ls_count; - } - - - /* free memory */ - free( (void *) image ); /* only the double_image structure should be freed, - the data pointer was provided to this functions - and should not be destroyed. */ - free_image_double(angles); - free_image_double(modgrad); - free_image_char(used); - free( (void *) reg ); - free( (void *) mem_p ); - - /* return the result */ - if( reg_img != NULL && reg_x != NULL && reg_y != NULL ) - { - if( region == NULL ) error("'region' should be a valid image."); - *reg_img = region->data; - if( region->xsize > (unsigned int) INT_MAX || - region->xsize > (unsigned int) INT_MAX ) - error("region image to big to fit in INT sizes."); - *reg_x = (int) (region->xsize); - *reg_y = (int) (region->ysize); - - /* free the 'region' structure. - we cannot use the function 'free_image_int' because we need to keep - the memory with the image data to be returned by this function. */ - free( (void *) region ); - } - if( out->size > (unsigned int) INT_MAX ) - error("too many detections to fit in an INT."); - *n_out = (int) (out->size); - - return_value = out->values; - free( (void *) out ); /* only the 'ntuple_list' structure must be freed, - but the 'values' pointer must be keep to return - as a result. */ - - return return_value; -} - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface with Scale and Region output. - */ -float * lsd_scale_region( int * n_out, - float * img, int X, int Y, float scale, - int ** reg_img, int * reg_x, int * reg_y ) -{ - /* LSD parameters */ - float sigma_scale = 0.6; /* Sigma for Gaussian filter is computed as - sigma = sigma_scale/scale. */ - float quant = 2.0; /* Bound to the quantization error on the - gradient norm. */ - float ang_th = 22.5; /* Gradient angle tolerance in degrees. */ - float log_eps = 0.0; /* Detection threshold: -log10(NFA) > log_eps */ - float density_th = 0.7; /* Minimal density of region points in rectangle. */ - int n_bins = 1024; /* Number of bins in pseudo-ordering of gradient - modulus. */ - - return LineSegmentDetection( n_out, img, X, Y, scale, sigma_scale, quant, - ang_th, log_eps, density_th, n_bins, - reg_img, reg_x, reg_y ); -} - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface with Scale. - */ -float * lsd_scale(int * n_out, float * img, int X, int Y, float scale) -{ - return lsd_scale_region(n_out,img,X,Y,scale,NULL,NULL,NULL); -} - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface. - */ -float * lsd(int * n_out, float * img, int X, int Y) -{ - /* LSD parameters */ - float scale = 0.8; /* Scale the image by Gaussian filter to 'scale'. */ - - return lsd_scale(n_out,img,X,Y,scale); - - -} -/*----------------------------------------------------------------------------*/ - -/***** added by manuel aristaran ****/ - -void free_values(float * p) { - free((void *) p); -} diff --git a/ext/lsd.h b/ext/lsd.h deleted file mode 100644 index 0e8315e..0000000 --- a/ext/lsd.h +++ /dev/null @@ -1,283 +0,0 @@ -/*---------------------------------------------------------------------------- - - LSD - Line Segment Detector on digital images - - This code is part of the following publication and was subject - to peer review: - - "LSD: a Line Segment Detector" by Rafael Grompone von Gioi, - Jeremie Jakubowicz, Jean-Michel Morel, and Gregory Randall, - Image Processing On Line, 2012. DOI:10.5201/ipol.2012.gjmr-lsd - http://dx.doi.org/10.5201/ipol.2012.gjmr-lsd - - Copyright (c) 2007-2011 rafael grompone von gioi - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - - ----------------------------------------------------------------------------*/ - -/*----------------------------------------------------------------------------*/ -/** @file lsd.h - LSD module header - @author rafael grompone von gioi - */ -/*----------------------------------------------------------------------------*/ -#ifndef LSD_HEADER -#define LSD_HEADER - -/*----------------------------------------------------------------------------*/ -/** LSD Full Interface - - @param n_out Pointer to an int where LSD will store the number of - line segments detected. - - @param img Pointer to input image data. It must be an array of - doubles of size X x Y, and the pixel at coordinates - (x,y) is obtained by img[x+y*X]. - - @param X X size of the image: the number of columns. - - @param Y Y size of the image: the number of rows. - - @param scale When different from 1.0, LSD will scale the input image - by 'scale' factor by Gaussian filtering, before detecting - line segments. - Example: if scale=0.8, the input image will be subsampled - to 80% of its size, before the line segment detector - is applied. - Suggested value: 0.8 - - @param sigma_scale When scale!=1.0, the sigma of the Gaussian filter is: - sigma = sigma_scale / scale, if scale < 1.0 - sigma = sigma_scale, if scale >= 1.0 - Suggested value: 0.6 - - @param quant Bound to the quantization error on the gradient norm. - Example: if gray levels are quantized to integer steps, - the gradient (computed by finite differences) error - due to quantization will be bounded by 2.0, as the - worst case is when the error are 1 and -1, that - gives an error of 2.0. - Suggested value: 2.0 - - @param ang_th Gradient angle tolerance in the region growing - algorithm, in degrees. - Suggested value: 22.5 - - @param log_eps Detection threshold, accept if -log10(NFA) > log_eps. - The larger the value, the more strict the detector is, - and will result in less detections. - (Note that the 'minus sign' makes that this - behavior is opposite to the one of NFA.) - The value -log10(NFA) is equivalent but more - intuitive than NFA: - - -1.0 gives an average of 10 false detections on noise - - 0.0 gives an average of 1 false detections on noise - - 1.0 gives an average of 0.1 false detections on nose - - 2.0 gives an average of 0.01 false detections on noise - . - Suggested value: 0.0 - - @param density_th Minimal proportion of 'supporting' points in a rectangle. - Suggested value: 0.7 - - @param n_bins Number of bins used in the pseudo-ordering of gradient - modulus. - Suggested value: 1024 - - @param reg_img Optional output: if desired, LSD will return an - int image where each pixel indicates the line segment - to which it belongs. Unused pixels have the value '0', - while the used ones have the number of the line segment, - numbered 1,2,3,..., in the same order as in the - output list. If desired, a non NULL int** pointer must - be assigned, and LSD will make that the pointer point - to an int array of size reg_x x reg_y, where the pixel - value at (x,y) is obtained with (*reg_img)[x+y*reg_x]. - Note that the resulting image has the size of the image - used for the processing, that is, the size of the input - image scaled by the given factor 'scale'. If scale!=1 - this size differs from XxY and that is the reason why - its value is given by reg_x and reg_y. - Suggested value: NULL - - @param reg_x Pointer to an int where LSD will put the X size - 'reg_img' image, when asked for. - Suggested value: NULL - - @param reg_y Pointer to an int where LSD will put the Y size - 'reg_img' image, when asked for. - Suggested value: NULL - - @return A double array of size 7 x n_out, containing the list - of line segments detected. The array contains first - 7 values of line segment number 1, then the 7 values - of line segment number 2, and so on, and it finish - by the 7 values of line segment number n_out. - The seven values are: - - x1,y1,x2,y2,width,p,-log10(NFA) - . - for a line segment from coordinates (x1,y1) to (x2,y2), - a width 'width', an angle precision of p in (0,1) given - by angle_tolerance/180 degree, and NFA value 'NFA'. - If 'out' is the returned pointer, the 7 values of - line segment number 'n+1' are obtained with - 'out[7*n+0]' to 'out[7*n+6]'. - */ -float * LineSegmentDetection( int * n_out, - float * img, int X, int Y, - float scale, float sigma_scale, float quant, - float ang_th, float log_eps, float density_th, - int n_bins, - int ** reg_img, int * reg_x, int * reg_y ); - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface with Scale and Region output. - - @param n_out Pointer to an int where LSD will store the number of - line segments detected. - - @param img Pointer to input image data. It must be an array of - doubles of size X x Y, and the pixel at coordinates - (x,y) is obtained by img[x+y*X]. - - @param X X size of the image: the number of columns. - - @param Y Y size of the image: the number of rows. - - @param scale When different from 1.0, LSD will scale the input image - by 'scale' factor by Gaussian filtering, before detecting - line segments. - Example: if scale=0.8, the input image will be subsampled - to 80% of its size, before the line segment detector - is applied. - Suggested value: 0.8 - - @param reg_img Optional output: if desired, LSD will return an - int image where each pixel indicates the line segment - to which it belongs. Unused pixels have the value '0', - while the used ones have the number of the line segment, - numbered 1,2,3,..., in the same order as in the - output list. If desired, a non NULL int** pointer must - be assigned, and LSD will make that the pointer point - to an int array of size reg_x x reg_y, where the pixel - value at (x,y) is obtained with (*reg_img)[x+y*reg_x]. - Note that the resulting image has the size of the image - used for the processing, that is, the size of the input - image scaled by the given factor 'scale'. If scale!=1 - this size differs from XxY and that is the reason why - its value is given by reg_x and reg_y. - Suggested value: NULL - - @param reg_x Pointer to an int where LSD will put the X size - 'reg_img' image, when asked for. - Suggested value: NULL - - @param reg_y Pointer to an int where LSD will put the Y size - 'reg_img' image, when asked for. - Suggested value: NULL - - @return A double array of size 7 x n_out, containing the list - of line segments detected. The array contains first - 7 values of line segment number 1, then the 7 values - of line segment number 2, and so on, and it finish - by the 7 values of line segment number n_out. - The seven values are: - - x1,y1,x2,y2,width,p,-log10(NFA) - . - for a line segment from coordinates (x1,y1) to (x2,y2), - a width 'width', an angle precision of p in (0,1) given - by angle_tolerance/180 degree, and NFA value 'NFA'. - If 'out' is the returned pointer, the 7 values of - line segment number 'n+1' are obtained with - 'out[7*n+0]' to 'out[7*n+6]'. - */ -float * lsd_scale_region( int * n_out, - float * img, int X, int Y, float scale, - int ** reg_img, int * reg_x, int * reg_y ); - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface with Scale - - @param n_out Pointer to an int where LSD will store the number of - line segments detected. - - @param img Pointer to input image data. It must be an array of - doubles of size X x Y, and the pixel at coordinates - (x,y) is obtained by img[x+y*X]. - - @param X X size of the image: the number of columns. - - @param Y Y size of the image: the number of rows. - - @param scale When different from 1.0, LSD will scale the input image - by 'scale' factor by Gaussian filtering, before detecting - line segments. - Example: if scale=0.8, the input image will be subsampled - to 80% of its size, before the line segment detector - is applied. - Suggested value: 0.8 - - @return A double array of size 7 x n_out, containing the list - of line segments detected. The array contains first - 7 values of line segment number 1, then the 7 values - of line segment number 2, and so on, and it finish - by the 7 values of line segment number n_out. - The seven values are: - - x1,y1,x2,y2,width,p,-log10(NFA) - . - for a line segment from coordinates (x1,y1) to (x2,y2), - a width 'width', an angle precision of p in (0,1) given - by angle_tolerance/180 degree, and NFA value 'NFA'. - If 'out' is the returned pointer, the 7 values of - line segment number 'n+1' are obtained with - 'out[7*n+0]' to 'out[7*n+6]'. - */ -float * lsd_scale(int * n_out, float * img, int X, int Y, float scale); - -/*----------------------------------------------------------------------------*/ -/** LSD Simple Interface - - @param n_out Pointer to an int where LSD will store the number of - line segments detected. - - @param img Pointer to input image data. It must be an array of - doubles of size X x Y, and the pixel at coordinates - (x,y) is obtained by img[x+y*X]. - - @param X X size of the image: the number of columns. - - @param Y Y size of the image: the number of rows. - - @return A double array of size 7 x n_out, containing the list - of line segments detected. The array contains first - 7 values of line segment number 1, then the 7 values - of line segment number 2, and so on, and it finish - by the 7 values of line segment number n_out. - The seven values are: - - x1,y1,x2,y2,width,p,-log10(NFA) - . - for a line segment from coordinates (x1,y1) to (x2,y2), - a width 'width', an angle precision of p in (0,1) given - by angle_tolerance/180 degree, and NFA value 'NFA'. - If 'out' is the returned pointer, the 7 values of - line segment number 'n+1' are obtained with - 'out[7*n+0]' to 'out[7*n+6]'. - */ -float * lsd(int * n_out, float * img, int X, int Y); - -void free_values(float * p); - -#endif /* !LSD_HEADER */ -/*----------------------------------------------------------------------------*/ diff --git a/lib/tabula.rb b/lib/tabula.rb index d9d773f..8823209 100644 --- a/lib/tabula.rb +++ b/lib/tabula.rb @@ -1,34 +1,13 @@ -module Tabula - PDFBOX = 'pdfbox-app-2.0.0-SNAPSHOT.jar' - ONLY_SPACES_RE = Regexp.new('^\s+$') -end - -require File.join(File.dirname(__FILE__), '../target/', Tabula::PDFBOX) -require File.join(File.dirname(__FILE__), '../target/', 'slf4j-api-1.6.3.jar') -require File.join(File.dirname(__FILE__), '../target/', 'trove4j-3.0.3.jar') -require File.join(File.dirname(__FILE__), '../target/', 'jsi-1.1.0-SNAPSHOT.jar') - - -import 'java.util.logging.LogManager' -import 'java.util.logging.Level' - -lm = LogManager.log_manager -lm.logger_names.each do |name| - if name == "" #rootlogger is apparently the logger PDFBox is talking to. - l = lm.get_logger(name) - l.level = Level::OFF - l.handlers.each do |h| - h.level = Level::OFF - end - end -end +require File.join(File.dirname(__FILE__), + '..', + 'target', + 'tabula-0.8.0-jar-with-dependencies.jar') +java.util.logging.Logger.getLogger('org.apache.pdfbox').setLevel(java.util.logging.Level::OFF) require_relative './tabula/version' require_relative './tabula/core_ext' + require_relative './tabula/entities' require_relative './tabula/extraction' -require_relative './tabula/table_extractor' -require_relative './tabula/writers' -require_relative './tabula/line_segment_detector' -require_relative './tabula/pdf_render' +require_relative './tabula/table_extractor' \ No newline at end of file diff --git a/lib/tabula/core_ext.rb b/lib/tabula/core_ext.rb index e93449a..62a3705 100644 --- a/lib/tabula/core_ext.rb +++ b/lib/tabula/core_ext.rb @@ -1,7 +1,19 @@ java_import java.awt.geom.Point2D java_import java.awt.geom.Line2D -java_import java.awt.geom.Rectangle2D -java_import java.awt.Rectangle + + +def debug_text_elements(text_elements) + require 'csv' + m = [:text, :top, :left, :bottom, :right, :width_of_space] + CSV($stderr) { |csv| + text_elements.each { |te| + csv << m.map { |method| + te.send(method) + } + } + } +end + class Array def rpad(padding, target_size) @@ -12,275 +24,3 @@ def rpad(padding, target_size) end end end - - -module Enumerable - - def sum - self.inject(0){|accum, i| accum + i } - end - - def mean - self.sum/self.length.to_f - end - - def sample_variance - m = self.mean - sum = self.inject(0) {|accum, i| accum + (i-m)**2 } - sum/(self.length - 1).to_f - end - - def standard_deviation - return Math.sqrt(self.sample_variance) - end - - def sorted? - each_cons(2).all? { |a, b| (a <=> b) <= 0 } - end - -end - -class Point2D::Float - def inspect - toString - end - - def to_json(*args) - [self.getX, self.getY].to_json(*args) - end - - def hash - "#{self.getX},#{self.getY}".hash - end - - def <=>(other) - return 1 if self.y > other.y - return -1 if self.y < other.y - return 1 if self.x > other.x - return -1 if self.x < other.x - return 0 - end - - def x_first_cmp(other) - return 1 if self.x > other.x - return -1 if self.x < other.x - return 1 if self.y > other.y - return -1 if self.y < other.y - return 0 - end - - def ==(other) - return self.x == other.x && self.y == other.y - end - -end - -class Line2D::Float - def to_json(*args) - [self.getX1, self.getY1, self.getX2, self.getY2].to_json(*args) - end - - def inspect - "" - end - - def rotate!(pointX, pointY, amount) - px1 = self.getX1 - pointX; px2 = self.getX2 - pointX - py1 = self.getY1 - pointY; py2 = self.getY2 - pointY - - if amount == 90 || amount == -270 - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], pointX - py2, pointY + px1, pointX - py1, pointY + px2 - elsif amount == 270 || amount == -90 - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], pointX + py1, pointY - px2, pointX + py2, pointY - px1 - end - - end - - def transform!(affine_transform) - newP1, newP2 = Point2D::Float.new, Point2D::Float.new - affine_transform.transform(self.getP1, newP1) - affine_transform.transform(self.getP2, newP2) - setLine(newP1, newP2) - self - end - - def snap!(cell_size) - newP1, newP2 = Point2D::Float.new, Point2D::Float.new - newP1.java_send :setLocation, [Java::float, Java::float], (self.getX1 / cell_size).round * cell_size, (self.getY1 / cell_size).round * cell_size - newP2.java_send :setLocation, [Java::float, Java::float], (self.getX2 / cell_size).round * cell_size, (self.getY2 / cell_size).round * cell_size - setLine(newP1, newP2) - end - - def horizontal?(threshold=0.00001) - (self.getY2 - self.getY1).abs < threshold - end - - def vertical?(threshold=0.00001) - (self.getX2 - self.getX1).abs < threshold - end - -end - -class Rectangle2D - SIMILARITY_DIVISOR = 20 - - alias_method :top, :minY - alias_method :right, :maxX - alias_method :left, :minX - alias_method :bottom, :maxY - - - # Implement geometry stuff - #------------------------- - - def dims(*format) - if format - format.map{|method| self.send(method)} - else - [self.x, self.y, self.width, self.height] - end - end - - def top=(new_y) - delta_height = new_y - self.y - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], self.x, new_y, self.width, (self.height - delta_height) - - #used to be: (fixes test_vertical_rulings_splitting_words) - # self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], self.x, new_y, self.width, self.height - end - - def bottom=(new_y2) - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], self.x, self.y, self.width, new_y2 - self.y - end - - def left=(new_x) - delta_width = new_x - self.x - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], new_x, self.y, (self.width - delta_width), self.height - #used to be: (fixes test_vertical_rulings_splitting_words) - # self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], new_x, self.y, self.width, self.height - end - - def right=(new_x2) - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], self.x, self.y, new_x2 - self.x, self.height - end - - def area - self.width * self.height - end - - # [x, y] - def midpoint - [horizontal_midpoint, vertical_midpoint] - end - - def horizontal_midpoint - self.left + (self.width / 2) - end - - def vertical_midpoint - self.top + (self.height / 2) - end - - def horizontal_distance(other) - (other.left - self.right).abs - end - - def vertical_distance(other) - (other.bottom - self.bottom).abs - end - - - # Various ways that rectangles can overlap one another - #------------------------------ - - # Roughly, detects if self and other belong to the same line - def vertically_overlaps?(other) - vertical_overlap = [0, [self.bottom, other.bottom].min - [self.top, other.top].max].max - vertical_overlap > 0 - end - - # detects if self and other belong to the same column - def horizontally_overlaps?(other) - horizontal_overlap = [0, [self.right, other.right].min - [self.left, other.left].max].max - horizontal_overlap > 0 - end - - def overlaps?(other) - self.intersects(*other.dims(:x, :y, :width, :height)) - end - - def overlaps_with_ratio?(other, ratio_tolerance=0.00001) - self.overlap_ratio(other) > ratio_tolerance - end - - def overlap_ratio(other) - intersection_width = [0, [self.right, other.right].min - [self.left, other.left].max].max - intersection_height = [0, [self.bottom, other.bottom].min - [self.top, other.top].max].max - intersection_area = [0, intersection_height * intersection_width].max - - union_area = self.area + other.area - intersection_area - intersection_area / union_area - end - - # as defined by PDF-TREX paper - def horizontal_overlap_ratio(other) - delta = [self.bottom - self.top, other.bottom - other.top].min - if [other.top, self.top, other.bottom, self.bottom].sorted? - (other.bottom - self.top) / delta - elsif [self.top, other.top, self.bottom, other.bottom].sorted? - (self.bottom - other.top) / delta - elsif [self.top, other.top, other.bottom, self.bottom].sorted? - (other.bottom - other.top) / delta - elsif [other.top, self.top, self.bottom, other.bottom].sorted? - (self.bottom - self.top) / delta - else - 0 - end - end - - - # Funky custom methods (i.e. not just geometry) - #---------------------------------------------- - - #used for "deduping" similar rectangles detected via CV. - def similarity_hash - [self.x.to_i / SIMILARITY_DIVISOR, self.y.to_i / SIMILARITY_DIVISOR, self.width.to_i / SIMILARITY_DIVISOR, self.height.to_i / SIMILARITY_DIVISOR].to_s - end - - def self.unionize(non_overlapping_rectangles, next_rect) - #if next_rect doesn't overlap any of non_overlapping_rectangles - if !(overlapping = non_overlapping_rectangles.compact.select{|r| next_rect.overlaps? r}).empty? && - !non_overlapping_rectangles.empty? - #remove all of those that it overlaps from non_overlapping_rectangles and - non_overlapping_rectangles -= overlapping - #add to non_overlapping_rectangles the bounding box of the overlapping rectangles. - non_overlapping_rectangles << overlapping.inject(next_rect) do |memo, overlap| - #all we're doing is unioning `overlap` and `memo` and setting that result to `memo` - union(overlap, memo, memo) #I " - end - -end - -# used only in GetBounds2D in an intermediate step in HasCells#find_spreadsheets_from_cells -class Rectangle #java.awt.Rectangle - def inspect - "#" - end -end diff --git a/lib/tabula/entities.rb b/lib/tabula/entities.rb index 6aafcf3..bb4ea57 100644 --- a/lib/tabula/entities.rb +++ b/lib/tabula/entities.rb @@ -1,10 +1,7 @@ -require_relative './entities/zone_entity' require_relative './entities/cell' require_relative './entities/has_cells' require_relative './entities/line' -require_relative './entities/text_element_index' require_relative './entities/page' -require_relative './entities/page_area' require_relative './entities/ruling' require_relative './entities/spreadsheet' require_relative './entities/table' diff --git a/lib/tabula/entities/cell.rb b/lib/tabula/entities/cell.rb index 8968638..82dbacd 100644 --- a/lib/tabula/entities/cell.rb +++ b/lib/tabula/entities/cell.rb @@ -1,55 +1,10 @@ module Tabula + Cell = Java::TechnologyTabula::Cell + class Java::TechnologyTabula::Cell + attr_accessor :options - #cells are components of spreadsheets - - class Cell < ZoneEntity - - NORMAL = 0 - DEBUG = 1 - SUPERDEBUG = 2 - - attr_accessor :text_elements, :placeholder, :spanning, :options - - def initialize(top, left, width, height, options={}) - super(top, left, width, height) - @placeholder = false - @spanning = false - @text_elements = [] - @options = ({:use_line_returns => true, :cell_debug => NORMAL}).merge options - end - - def self.new_from_points(topleft, bottomright, options={}) - width = bottomright.x - topleft.x - height = bottomright.y - topleft.y - Cell.new(topleft.y, topleft.x, width, height, options) - end - - def text - return "placeholder" if @placeholder && @options[:cell_debug] >= DEBUG - output = "" - text_elements.sort #use the default sort for ZoneEntity - text_elements.group_by(&:top).values.each do |row| - output << row.map{|el| el.text}.join('') + (@options[:use_line_returns] ? "\r" : '') - # per @bchartoff, https://github.com/jazzido/tabula-extractor/pull/65#issuecomment-32899336 - # line returns as \r behave better in Excel. - end - if (output.empty? && @options[:cell_debug] >= DEBUG) || @options[:cell_debug] >= SUPERDEBUG - text_output = output.dup - output = "top: #{top} left: #{left} \n w: #{width} h: #{height}" - output += " \n #{text_output}" - end - output.strip - end - - def to_json(*a) - { - 'json_class' => self.class.name, - 'text' => text, - 'top' => top, - 'left' => left, - 'width' => width, - 'height' => height - }.to_json(*a) + def text(use_line_returns=nil) + java_send(:getText, [Java::boolean], use_line_returns.nil? ? (options.nil? || options[:use_line_returns].nil? ? true : options[:use_line_returns]) : use_line_returns) end end end diff --git a/lib/tabula/entities/has_cells.rb b/lib/tabula/entities/has_cells.rb index a14766f..1783f13 100644 --- a/lib/tabula/entities/has_cells.rb +++ b/lib/tabula/entities/has_cells.rb @@ -1,244 +1,29 @@ require 'set' java_import java.awt.Polygon java_import java.awt.geom.Area +java_import Java::TechnologyTabulaExtractors::SpreadsheetExtractionAlgorithm module Tabula # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors; ruling_lines reader module HasCells - ANOTHER_MAGIC_NUMBER = 0.75 + ARBITRARY_MAGIC_HEURISTIC_NUMBER = 0.65 def is_tabular? - #spreadsheet extraction - spreadsheet = spreadsheets.first - return false if spreadsheet.nil? - rows_defined_by_lines = spreadsheet.rows.size #rows filled in automatically - columns_defined_by_lines = spreadsheet.cols.size - - table = self.get_table - columns_defined_without_lines = table.cols.size - rows_defined_without_lines = table.rows.size - ratio = ((columns_defined_by_lines.to_f / columns_defined_without_lines) + (rows_defined_by_lines.to_f / rows_defined_without_lines)) / 2 - - return ratio > ANOTHER_MAGIC_NUMBER && ratio < (1 / ANOTHER_MAGIC_NUMBER) + SpreadsheetExtractionAlgorithm.new.isTabular(self) end - # finds cells from the ruling lines on the page. - # implements Nurminen thesis algorithm cf. https://github.com/jazzido/tabula-extractor/issues/16 - # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors - def find_cells!(options={}) - # All lines need to been sorted from up to down, - # and left to right in ascending order - - cellsFound = [] - - intersection_points = Ruling.find_intersections(horizontal_ruling_lines, vertical_ruling_lines) - - # All crossing-points have been sorted from up to down, - # and left to right in ascending order - # depending on the Point2D default sort here. - intersection_points_array = intersection_points.keys.sort - - intersection_points_array.each_with_index do |topLeft, i| - # Fetch all points on the same vertical and horizontal - # line with current crossing point - horizontal, vertical = intersection_points[topLeft] - - # this lets us go to the next intersection_point in intersection_points_array - # it is bad and I feel bad. - catch :cellCreated do - - # CrossingPointsDirectlyBelow( topLeft ); - x_points = intersection_points_array[i..-1].select{|pt| pt.x == topLeft.x && pt.y > topLeft.y } - # CrossingPointsDirectlyToTheRight( topLeft ); - y_points = intersection_points_array[i..-1].select{|pt| pt.y == topLeft.y && pt.x > topLeft.x } - - - x_points.each do |x_point| - # Skip to next crossing-point - # if( NOT EdgeExistsBetween( topLeft, x_point)) next crossing- - # point; - next unless vertical.colinear?(x_point) - y_points.each do |y_point| - - # if( NOT EdgeExistsBetween( topLeft, y_point)) next crossing- - # point; - next unless horizontal.colinear?(y_point) - #Hypothetical bottom right point of rectangle - btmRight = Point2D::Float.new(y_point.x, x_point.y) - if intersection_points.include?(btmRight) - btmRightHorizontal, btmRightVertical = intersection_points[btmRight] - - if btmRightHorizontal.colinear?( x_point ) && - btmRightVertical.colinear?( y_point ) - # Rectangle is confirmed to have 4 sides - cellsFound << Cell.new_from_points( topLeft, btmRight, options) - # Each crossing point can be the top left corner - # of only a single rectangle - #next crossing-point; we need to "next" out of the outer loop here - # to avoid creating non-minimal cells, I htink. - throw :cellCreated - end - end - end - end - end #cellCreated - end - self.cells = cellsFound - cellsFound - end - - ############################# - # Chapter 2, Spanning Cells # - ############################# - #if c is a "spanning cell", that is - # if there are N>0 vertical lines strictly between this cell's left and right - #insert N placeholder cells after it with zero size (but same top) - - # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors - def add_spanning_cells! - #rounding: because Cell.new_from_points, using in #find_cells above, has - # a float precision error where, for instance, a cell whose x2 coord is - # supposed to be 160.137451171875 comes out as 160.13745498657227 because - # of minus. :( - vertical_uniq_locs = vertical_ruling_lines.map{|l| l.left.round(5)}.uniq #already sorted - horizontal_uniq_locs = horizontal_ruling_lines.map{|l| l.top.round(5)}.uniq #already sorted - - cells.each do |c| - vertical_rulings_spanned_over = vertical_uniq_locs.select{|l| l > c.left.round(5) && l < c.right.round(5) } - horizontal_rulings_spanned_over = horizontal_uniq_locs.select{|t| t > c.top.round(5) && t < c.bottom.round(5) } - - unless vertical_rulings_spanned_over.empty? - c.spanning = true - vertical_rulings_spanned_over.each do |spanned_over_line_loc| - placeholder = Cell.new(c.top, spanned_over_line_loc, 0, c.height) - placeholder.placeholder = true - cells << placeholder - end - end - unless horizontal_rulings_spanned_over.empty? - c.spanning = true - horizontal_rulings_spanned_over.each do |spanned_over_line_loc| - placeholder = Cell.new(spanned_over_line_loc, c.left, c.width, 0) - placeholder.placeholder = true - cells << placeholder - end - end - - #if there's a spanning cell that's spans over both rows and columns, then it has "double placeholder" cells - # e.g. ------------------- - # | C | C | C | C | (this is some pretty sweet ASCII art, eh?) - # |-----------------| - # | C | C | C | C | - # |-----------------| - # | C | SC P | C | where MC is the "spanning cell" that holds all the text within its bounds - # |---- + ----| P is a "placeholder" cell with either zero width or zero height - # | C | P DP | C | DP is a "double placeholder" cell with zero width and zero height - # |---- + ----| C is an ordinary cell. - # | C | P DP | C | - # |-----------------| - - unless (double_placeholders = vertical_rulings_spanned_over.product(horizontal_rulings_spanned_over)).empty? - double_placeholders.each do |vert_spanned_over, horiz_spanned_over| - placeholder = Cell.new(horiz_spanned_over, vert_spanned_over, 0, 0) - placeholder.placeholder = true - cells << placeholder - end - end - end + def find_cells!(horizontal_ruling_lines, vertical_ruling_lines, options={}) + self.cells = SpreadsheetExtractionAlgorithm.findCells(horizontal_ruling_lines, vertical_ruling_lines) end #TODO: #returns array of Spreadsheet objects constructed (or spreadsheet_areas => cells) #maybe placeholders should be added after cells is split into spreadsheets - def find_spreadsheets_from_cells - cells.sort! - - # via http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon - - points = Set.new - cells.each do |cell| - #TODO: keep track of cells for each point here for more efficiently keeping track of cells inside a polygon - cell.points.each do |pt| - if points.include?(pt) # Shared vertex, remove it. - points.delete(pt) - else - points << pt - end - end - end - points = points.to_a - - #x first sort - points_sort_x = points.sort{ |s, other| s.x_first_cmp(other) } - points_sort_y = points.sort - - edges_h = {} - edges_v = {} - i = 0 - while i < points.size do - curr_y = points_sort_y[i].y - while i < points.size && points_sort_y[i].y == curr_y do - edges_h[points_sort_y[i]] = points_sort_y[i + 1] - edges_h[points_sort_y[i + 1]] = points_sort_y[i] - i += 2 - end - end - - i = 0 - while i < points.size do - curr_x = points_sort_x[i].x - while i < points.size && points_sort_x[i].x == curr_x do - edges_v[points_sort_x[i]] = points_sort_x[i + 1] - edges_v[points_sort_x[i + 1]] = points_sort_x[i] - i += 2 - end - end - - # Get all the polygons. - polygons = [] - while !edges_h.empty? - # We can start with any point. - #TODO: should the polygon be represented just by an ordered array of points? - polygon = [[edges_h.shift[0], :horiz]] #popitem removes and returns a random key-value pair - loop do - curr, e = polygon.last - if e == :horiz - next_vertex = edges_v.delete(curr) - polygon << [next_vertex, :vert] - else - next_vertex = edges_h.delete(curr) #pop removes and returns the value at key `curr` - polygon << [next_vertex, :horiz] - end - if polygon[-1] == polygon[0] - # Closed polygon - polygon.pop() - break - end - end - - # Remove implementation-markers (:horiz and :vert) from the polygon. - polygon.map!{|point, _| point} - polygon.each do |vertex| - edges_h.delete(vertex) if edges_h.include?(vertex) - edges_v.delete(vertex) if edges_v.include?(vertex) - end - polygons << polygon - end - - # for efficiency's sake, we maybe ought to use java Polygon objects internally - # for flexibility, we don't. - - polygons.map do |polygon| - xpoints = [] - ypoints = [] - polygon.each do |pt| - xpoints << pt.x - ypoints << pt.y - end - Area.new(Polygon.new(xpoints.to_java(Java::int), ypoints.to_java(Java::int), xpoints.size)) #lol jruby - end + def find_spreadsheets_from_cells + SpreadsheetExtractionAlgorithm.new.findSpreadsheetsFromCells(self.cells) end + end end diff --git a/lib/tabula/entities/line.rb b/lib/tabula/entities/line.rb index 5166b9f..7debd42 100644 --- a/lib/tabula/entities/line.rb +++ b/lib/tabula/entities/line.rb @@ -1,39 +1,14 @@ -module Tabula - class Line < ZoneEntity - attr_accessor :text_elements +class Tabula::Line < Java::TechnologyTabula::Line attr_reader :index - def initialize(index=nil) - @text_elements = [] - @index = index - end - - def <<(t) - if @text_elements.size == 0 - @text_elements << t - self.top = t.top - self.left = t.left - self.width = t.width - self.height = t.height - else - if in_same_column = @text_elements.find { |te| te.horizontally_overlaps?(t) } - in_same_column.merge!(t) - else - self.text_elements << t - self.merge!(t) - end - end - end - #used for testing, ignores text element stuff besides stripped text. def ==(other) return false if other.nil? - self.text_elements = self.text_elements.rpad(TextElement::EMPTY, other.text_elements.size) - other.text_elements = other.text_elements.rpad(TextElement::EMPTY, self.text_elements.size) + self.text_elements = self.text_elements.rpad(Tabula::TextElement::EMPTY, other.text_elements.size) + other.text_elements = other.text_elements.rpad(Tabula::TextElement::EMPTY, self.text_elements.size) self.text_elements.zip(other.text_elements).inject(true) do |memo, my_yours| my, yours = my_yours memo && my == yours end end - end end diff --git a/lib/tabula/entities/page.rb b/lib/tabula/entities/page.rb index 28803d1..ff5972b 100644 --- a/lib/tabula/entities/page.rb +++ b/lib/tabula/entities/page.rb @@ -1,260 +1,78 @@ -module Tabula - class Page < ZoneEntity - include Tabula::HasCells - - attr_reader :rotation, :number_one_indexed, :file_path - attr_writer :min_char_width, :min_char_height - attr_accessor :cells - - def initialize(file_path, width, height, rotation, number, texts=[], ruling_lines=[], min_char_width=nil, min_char_height=nil) - super(0, 0, width, height) - @rotation = rotation - if number < 1 - raise ArgumentError, "Tabula::Page numbers are one-indexed; numbers < 1 are invalid." - end - @ruling_lines = ruling_lines - @file_path = file_path - @number_one_indexed = number - @cells = [] - @spreadsheets = nil - @min_char_width = min_char_width - @min_char_height = min_char_height - @spatial_index = TextElementIndex.new - - self.texts = texts - self.texts.each { |te| @spatial_index << te } - end - - def min_char_width - @min_char_width ||= texts.map(&:width).min - end - - def min_char_height - @min_char_height ||= texts.map(&:height).min - end +java_import Java::TechnologyTabula::Page +java_import Java::TechnologyTabulaExtractors::BasicExtractionAlgorithm +java_import Java::TechnologyTabulaExtractors::SpreadsheetExtractionAlgorithm - def get_area(area) - if area.is_a?(Array) - top, left, bottom, right = area - area = Tabula::ZoneEntity.new(top, left, - right - left, bottom - top) - end +class Page + include Tabula::HasCells + attr_accessor :file_path, :cells - texts = self.get_text(area) - page_area = PageArea.new(file_path, - area.width, - area.height, - rotation, - number, - texts, - Ruling.crop_rulings_to_area(@ruling_lines, area), - texts.map(&:width).min, - texts.map(&:height).min) - return page_area - end - - #returns a Table object - def get_table(options={}) - options = {:vertical_rulings => []}.merge(options) - if texts.empty? - return Tabula::Table.new(0, []) - end + #returns a Table object + def get_table(options={}) + options = {:vertical_rulings => []}.merge(options) - text_chunks = TextElement.merge_words(self.texts.sort, options).sort + tables = if options[:vertical_rulings].empty? + BasicExtractionAlgorithm.new.extract(self) + else + BasicExtractionAlgorithm.new(options[:vertical_rulings]).extract(self) + end - lines = TextChunk.group_by_lines(text_chunks) - - unless options[:vertical_rulings].empty? - columns = options[:vertical_rulings].map(&:left) #pixel locations, not entities - separators = columns.sort.reverse - else - columns = TextChunk.column_positions(lines) - separators = columns[1..-1].sort.reverse - end - - table = Table.new(lines.count, separators) - lines.each_with_index do |line, i| - line.text_elements.each do |te| - j = separators.find_index { |s| te.left > s } || separators.count - table.add_text_element(te, i, separators.count - j) - end - end - - table.lstrip_lines! - table - end - - #for API backwards-compatibility reasons, this returns an array of arrays. - def make_table(options={}) - get_table(options).rows - end - - # returns the Spreadsheets; creating them if they're not memoized - def spreadsheets(options={}) - unless @spreadsheets.nil? - return @spreadsheets - end - get_ruling_lines!(options) - self.find_cells!(options) - - spreadsheet_areas = find_spreadsheets_from_cells #literally, java.awt.geom.Area objects. lol sorry. polygons. - - #transform each spreadsheet area into a rectangle - # and get the cells contained within it. - spreadsheet_rectangle_areas = spreadsheet_areas.map{|a| a.getBounds } #getBounds2D is theoretically better, but returns a Rectangle2D.Double, which doesn't have our Ruby sugar on it. - - @spreadsheets = spreadsheet_rectangle_areas.map do |rect| - spr = Spreadsheet.new(rect.y, rect.x, - rect.width, rect.height, - self, - #TODO: keep track of the cells, instead of getting them again inefficiently. - [], - vertical_ruling_lines.select{|vl| rect.intersectsLine(vl) }, - horizontal_ruling_lines.select{|hl| rect.intersectsLine(hl) } - ) - spr.cells = @cells.select{|c| spr.overlaps?(c) } - spr.add_spanning_cells! - spr - end - if options[:fill_in_cells] - fill_in_cells! - end - spreadsheets - end - - def fill_in_cells!(options={}) - spreadsheets(options).each do |spreadsheet| - spreadsheet.cells.each do |cell| - cell.text_elements = page.get_cell_text(cell) - end - spreadsheet.cells_resolved = true - end - end - - def number(indexing_base=:one_indexed) - if indexing_base == :zero_indexed - return @number_one_indexed - 1 - else - return @number_one_indexed - end - end - - # TODO no need for this, let's choose one name - def ruling_lines - get_ruling_lines! - end - - def horizontal_ruling_lines - get_ruling_lines! - @horizontal_ruling_lines.nil? ? [] : @horizontal_ruling_lines - end - - def vertical_ruling_lines - get_ruling_lines! - @vertical_ruling_lines.nil? ? [] : @vertical_ruling_lines - end - - #returns ruling lines, memoizes them in - def get_ruling_lines!(options={}) - if @ruling_lines.nil? || @ruling_lines.empty? - return [] - end - self.snap_points! - - @ruling_lines.select! { |l| !(l.width == 0 && l.height == 0) } - - @vertical_ruling_lines ||= Ruling.collapse_oriented_rulings(@ruling_lines.select(&:vertical?)) - @horizontal_ruling_lines ||= Ruling.collapse_oriented_rulings(@ruling_lines.select(&:horizontal?)) - - @vertical_ruling_lines + @horizontal_ruling_lines + tables.first + end - end + #for API backwards-compatibility reasons, this returns an array of arrays. + def make_table(options={}) + get_table(options).rows + end - ## - # get text insidea area - # area can be an Array ([top, left, width, height]) - # or a Rectangle2D - def get_text(area=nil) - if area.instance_of?(Array) - top, left, bottom, right = area - area = Tabula::ZoneEntity.new(top, left, - right - left, bottom - top) - end - if area.nil? - texts - else - @spatial_index.contains(area) - end + # returns the Spreadsheets; creating them if they're not memoized + def spreadsheets(options={}) + unless @spreadsheets.nil? + return @spreadsheets end + SpreadsheetExtractionAlgorithm.new.extract(self).to_a.sort # to_a converts from java.util.ArrayList to Ruby Array + end - def fill_in_cell_texts!(areas) - texts.each do |t| - area = areas.find{|a| a.contains(t) } - area.text_elements << t unless area.nil? - end - areas.each do |area| - area.text_elements = TextElement.merge_words(area.text_elements) + def fill_in_cells!(options={}) + spreadsheets(options).each do |spreadsheet| + spreadsheet.cells.each do |cell| + cell.text_elements = page.get_cell_text(cell) end + spreadsheet.cells_resolved = true end + end - def get_cell_text(area=nil) - TextElement.merge_words(self.get_text(area)) - end - - def to_json(options={}) - { :width => self.width, - :height => self.height, - :number => self.number, - :rotation => self.rotation, - :texts => self.texts - }.to_json(options) - end + def number(indexing_base=:one_indexed) + # if indexing_base == :zero_indexed + # return @number_one_indexed - 1 + # else + # return @number_one_indexed + # end + self.page_number + end - def snap_points! - lines_to_points = {} - points = [] - @ruling_lines.each do |line| - point1 = line.p1 #comptooters are the wurst - point2 = line.p2 - # for a given line, each call to #p1 and #p2 creates a new - # Point2D::Float object, rather than returning the same one over and - # over again. - # so we have to get it, store it in memory as `point1` and `point2` - # and then store those in various places (and now, modifying one will - # modify the reference and thereby modify the other) - lines_to_points[line] = [point1, point2] - points += [point1, point2] - end + # TODO no need for this, let's choose one name + def ruling_lines + get_ruling_lines! + end - # lines are stored separately from their constituent points - # so you can't modify the points and then modify the lines. - # ah, but perhaps I can stick the points in a hash AND in an array - # and then modify the lines by means of the points in the hash. + def horizontal_ruling_lines + self.getHorizontalRulings + end - [[:x, :x=, self.min_char_width], [:y, :y=, self.min_char_height]].each do |getter, setter, cell_size| - sorted_points = points.sort_by(&getter) - first_point = sorted_points.shift - grouped_points = sorted_points.inject([[first_point]] ) do |memo, next_point| - last = memo.last + def vertical_ruling_lines + self.getVerticalRulings + end - if (next_point.send(getter) - last.first.send(getter)).abs < cell_size - memo[-1] << next_point - else - memo << [next_point] - end - memo - end - grouped_points.each do |group| - uniq_locs = group.map(&getter).uniq - avg_loc = uniq_locs.sum / uniq_locs.size - group.each{|p| p.send(setter, avg_loc) } - end - end + #returns ruling lines, memoizes them in + def get_ruling_lines! + self.get_rulings + end - lines_to_points.each do |l, p1_p2| - l.java_send :setLine, [java.awt.geom.Point2D, java.awt.geom.Point2D], p1_p2[0], p1_p2[1] - end - end + def get_cell_text(area=nil) + self.get_text(area) end +end +module Tabula + Page = ::Page end diff --git a/lib/tabula/entities/page_area.rb b/lib/tabula/entities/page_area.rb index 6aab838..64789fd 100644 --- a/lib/tabula/entities/page_area.rb +++ b/lib/tabula/entities/page_area.rb @@ -1,7 +1,6 @@ module Tabula class PageArea < Page - end end diff --git a/lib/tabula/entities/ruling.rb b/lib/tabula/entities/ruling.rb index 7bf3e05..57db9aa 100644 --- a/lib/tabula/entities/ruling.rb +++ b/lib/tabula/entities/ruling.rb @@ -1,358 +1,12 @@ -module Tabula - class Ruling < java.awt.geom.Line2D::Float +class Tabula::Ruling < Java::TechnologyTabula::Ruling - attr_accessor :stroking_color + # some PDFs (garment factory audits, precise link TK) make tables by drawing lines that + # very nearly intersect each other, but not quite. E.g. a horizontal line spans the table at a Y val of 100 + # and each vertical line (i.e. column separating ruling line) starts at 101 or 102. + # this is very annoying. so we check if those lines nearly overlap by expanding each pair + # by 2 pixels in each direction (so the vertical lines' top becomes 99 or 100, and then the expanded versions overlap) - def initialize(top, left, width, height, stroking_color=nil) - super(left, top, left+width, top+height) - self.stroking_color = stroking_color - end - - alias :top :getY1 - alias :left :getX1 - alias :bottom :getY2 - alias :right :getX2 - - def top=(v) - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, v, right, bottom - end - - def left=(v) - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], v, top, right, bottom - end - - def bottom=(v) - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, top, right, v - end - - def right=(v) - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, top, v, bottom - end - - def width - right - left - end - - def height - bottom - top - end - - # attributes that make sense only for non-oblique lines - # these are used to have a single collapse method (in page, currently) - def position - raise NoMethodError, "Oblique line #{self.inspect} has no #position method." if oblique? - vertical? ? left : top - end - def start - raise NoMethodError, "Oblique line #{self.inspect} has no #start method." if oblique? - vertical? ? top : left - end - def end - raise NoMethodError, "Oblique line #{self.inspect} has no #end method." if oblique? - vertical? ? bottom : right - end - def position=(coord) - raise NoMethodError, "Oblique line #{self.inspect} has no #position= method." if oblique? - if vertical? - self.left = coord - self.right = coord - else - self.top = coord - self.bottom = coord - end - end - def start=(coord) - raise NoMethodError, "Oblique line #{self.inspect} has no #start= method." if oblique? - if vertical? - self.top = coord - else - self.left = coord - end - end - def end=(coord) - raise NoMethodError, "Oblique line #{self.inspect} has no #end= method." if oblique? - if vertical? - self.bottom = coord - else - self.right = coord - end - end - - #ok wtf are you doing, Jeremy? - # some PDFs (garment factory audits, precise link TK) make tables by drawing lines that - # very nearly intersect each other, but not quite. E.g. a horizontal line spans the table at a Y val of 100 - # and each vertical line (i.e. column separating ruling line) starts at 101 or 102. - # this is very annoying. so we check if those lines nearly overlap by expanding each pair - # by 2 pixels in each direction (so the vertical lines' top becomes 99 or 100, and then the expanded versions overlap) - - PERPENDICULAR_PIXEL_EXPAND_AMOUNT = 2 - COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT = 1 - - # if the lines we're comparing are colinear or parallel, we expand them by a only 1 pixel, - # because the expansions are additive - # (e.g. two vertical lines, at x = 100, with one having y2 of 98 and the other having y1 of 102 would - # erroneously be said to nearlyIntersect if they were each expanded by 2 (since they'd both terminate at 100). - # The COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT is only 1 so the total expansion is 2. - # A total expansion amount of 2 is empirically verified to work sometimes. It's not a magic number from any - # source other than a little bit of experience.) - - def nearlyIntersects?(another) - if self.intersectsLine(another) - true - elsif self.perpendicular_to?(another) - self.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)) - else - self.expand(COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT).intersectsLine(another.expand(COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT)) - end - end - - ## - # intersect this Ruling with a java.awt.geom.Rectangle2D - def intersect(area) - i = self.getBounds2D.createIntersection(area) - self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], i.getX, i.getY, i.getX + i.getWidth, i.getY + i.getHeight - self - end - - def expand(amt) - raise NoMethodError, "Oblique line #{self.inspect} has no #expand method." if oblique? - r = Ruling.new(self.top, self.left, self.width, self.height) - r.start = r.start - amt - r.end = r.end + amt - r - end - - - def length - Math.sqrt( (self.right - self.left).abs ** 2 + (self.bottom - self.top).abs ** 2 ) - end - - def vertical? - left == right - end - - def horizontal? - top == bottom - end - - def oblique? - !(vertical? || horizontal?) - end - - def perpendicular_to?(other) - return self.vertical? == other.horizontal? - end - - def to_json(arg) - [left, top, right, bottom].to_json - end - - def colinear?(point) - point.x >= left && point.x <= right && - point.y >= top && point.y <= bottom - end - - def ==(other) - return self.getX1 == other.getX1 && self.getY1 == other.getY1 && self.getX2 == other.getX2 && self.getY2 == other.getY2 - end - - ## - # calculate the intersection point between +self+ and other Ruling - def intersection_point(other) - # algo taken from http://mathworld.wolfram.com/Line-LineIntersection.html - - #self and other should always be perpendicular, since one should be horizontal and one should be vertical - self_l = self.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT) - other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT) - - return nil if !self_l.intersectsLine(other_l) - - horizontal, vertical = if self_l.horizontal? && other_l.vertical? - [self_l, other] - elsif self_l.vertical? && other_l.horizontal? - [other_l, self_l] - else - raise ArgumentError, "must be orthogonal, horizontal and vertical" - end - - - java.awt.geom.Point2D::Float.new(vertical.getX1, horizontal.getY1) - - end - - class HSegmentComparator - java_implements java.util.Comparator - def compare(o1, o2) - o1.top <=> o2.top - end - end - - ## - # log(n) implementation of find_intersections - # based on http://people.csail.mit.edu/indyk/6.838-old/handouts/lec2.pdf - def self.find_intersections(horizontals, verticals) - #tree = java.util.TreeMap.java_send(:initiailze, [COMP_CLASS], HSegmentComparator.new) - tree = java.util.TreeMap.new(HSegmentComparator.new) - sort_obj = Struct.new(:type, :pos, :obj) - - (horizontals + verticals) - .flat_map { |r| - r.vertical? ? sort_obj.new(:v, r.left, r) : [sort_obj.new(:hl, r.left, r), - sort_obj.new(:hr, r.right, r)] - } - .sort { |a,b| - if a.pos == b.pos - if a.type == :v && b.type == :hl - 1 - elsif a.type == :v && b.type == :hr - -1 - elsif a.type == :hl && b.type == :v - -1 - elsif a.type == :hr && b.type == :v - 1 - else - a.pos <=> b.pos - end - else - a.pos <=> b.pos - end - } - .inject({}) { |memo, e| - case e.type - when :v - tree.each { |h,_| - i = h.intersection_point(e.obj) - next memo if i.nil? - memo[i] = [h.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT), - e.obj.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)] - } - when :hr - tree.remove(e.obj) - when :hl - tree[e.obj] = 1 - end - memo - } - end - - ## - # crop an enumerable of +Ruling+ to an +area+ - def self.crop_rulings_to_area(rulings, area) - rulings.reduce([]) do |memo, r| - if r.intersects(area) - memo << r.clone.intersect(area) - end - memo - end - end - - def self.collapse_oriented_rulings(lines) - # lines must all be of one orientation (i.e. horizontal, vertical) - - if lines.empty? - return [] - end - - lines.sort! {|a, b| a.position != b.position ? a.position <=> b.position : a.start <=> b.start } - - lines = lines.inject([lines.shift]) do |memo, next_line| - last = memo.last - if next_line.position == last.position && last.nearlyIntersects?(next_line) - memo.last.start = next_line.start < last.start ? next_line.start : last.start - memo.last.end = next_line.end < last.end ? last.end : next_line.end - memo - elsif next_line.length == 0 - memo - else - memo << next_line - end - end - end - - # TODO do we really need this one anymore? - def self.clean_rulings(rulings, max_distance=4) - - # merge horizontal and vertical lines - # TODO this should be iterative - - skip = false - - horiz = rulings.select { |r| r.horizontal? } - .group_by(&:top) - .values.reduce([]) do |memo, rs| - - rs = rs.sort_by(&:left) - if rs.size > 1 - memo += - rs.each_cons(2) - .chunk { |p| p[1].left - p[0].right < 7 } - .select { |c| c[0] } - .map { |group| - group = group.last.flatten.uniq - Tabula::Ruling.new(group[0].top, - group[0].left, - group[-1].right - group[0].left, - 0) - } - Tabula::Ruling.new(rs[0].top, rs[0].left, rs[-1].right - rs[0].left, 0) - else - memo << rs.first - end - memo - end - .sort_by(&:top) - - h = [] - horiz.size.times do |i| - - if i == horiz.size - 1 - h << horiz[-1] - break - end - - if skip - skip = false; - next - end - d = (horiz[i+1].top - horiz[i].top).abs - - h << if d < max_distance # THRESHOLD DISTANCE between horizontal lines - skip = true - Tabula::Ruling.new(horiz[i].top + d / 2, [horiz[i].left, horiz[i+1].left].min, [horiz[i+1].width.abs, horiz[i].width.abs].max, 0) - else - horiz[i] - end - end - horiz = h - - vert = rulings.select { |r| r.vertical? } - .group_by(&:left) - .values - .reduce([]) do |memo, rs| - - rs = rs.sort_by(&:top) - - if rs.size > 1 - # Here be dragons: - # merge consecutive segments of lines that are close enough - memo += - rs.each_cons(2) - .chunk { |p| p[1].top - p[0].bottom < 7 } - .select { |c| c[0] } - .map { |group| - group = group.last.flatten.uniq - Tabula::Ruling.new(group[0].top, - group[0].left, - 0, - group[-1].bottom - group[0].top) - } - else - memo << rs.first - end - memo - end.sort_by(&:left) - - return horiz += vert - end + def to_json(arg) + [left, top, right, bottom].to_json end end diff --git a/lib/tabula/entities/spreadsheet.rb b/lib/tabula/entities/spreadsheet.rb index 2f26cc6..79c2daa 100644 --- a/lib/tabula/entities/spreadsheet.rb +++ b/lib/tabula/entities/spreadsheet.rb @@ -1,110 +1,103 @@ -module Tabula - # a counterpart of Table, to be sure. - # not sure yet what their relationship ought to be. +#java_import Java::TechnologyTabula::TableWithRulingLines - # the both should implement `cells`, `rows`, `cols`, `extraction_method` +class Java::TechnologyTabula::TableWithRulingLines + attr_accessor :vertical_ruling_lines, :horizontal_ruling_lines, :cells_resolved + attr_reader :extraction_method, :page - class Spreadsheet < ZoneEntity - include Tabula::HasCells - attr_accessor :cells, :vertical_ruling_lines, :horizontal_ruling_lines, :cells_resolved - attr_reader :extraction_method, :page + def self.empty(page) + Spreadsheet.new(0, 0, 0, 0, page, [], nil, nil) + end - def initialize(top, left, width, height, page, cells, vertical_ruling_lines, horizontal_ruling_lines) #, lines) - super(top, left, width, height) - @cells = cells - @page = page - @vertical_ruling_lines = vertical_ruling_lines - @horizontal_ruling_lines = horizontal_ruling_lines - @extraction_method = "spreadsheet" - end + # def ruling_lines=(lines) + # @vertical_ruling_lines = lines.select{|vl| vl.vertical? && spr.intersectsLine(vl) } + # @horizontal_ruling_lines = lines.select{|hl| hl.horizontal? && spr.intersectsLine(hl) } + # end - def ruling_lines - @vertical_ruling_lines + @horizontal_ruling_lines + # call `cols` with `evaluate_cells` as `false` to defer filling in the text in + # each cell, which can be computationally intensive. + def cols(evaluate_cells=true) + if evaluate_cells + fill_in_cells! end - - def ruling_lines=(lines) - @vertical_ruling_lines = lines.select{|vl| vl.vertical? && spr.intersectsLine(vl) } - @horizontal_ruling_lines = lines.select{|hl| hl.horizontal? && spr.intersectsLine(hl) } + lefts = cells.map(&:left).uniq.sort + lefts.map do |left| + cells.select{|c| c.left == left }.sort_by(&:top) end + end - def fill_in_cells! - unless @cells_resolved - @cells_resolved = true - cells.each do |cell| - cell.text_elements = @page.get_cell_text(cell) - end - end - end + # I don't think this is ever used in the new, thin-wrapper version of tabula-extractor. (that is, its functionality is contained entirely in tabula-java ) + # ####################################################### + # # Chapter 2 of Spreadsheet extraction, Spanning Cells # + # ####################################################### + # #if c is a "spanning cell", that is + # # if there are N>0 vertical lines strictly between this cell's left and right + # #insert N placeholder cells after it with zero size (but same top) + # def add_spanning_cells! + # #rounding: because Cell.new_from_points, using in #find_cells above, has + # # a float precision error where, for instance, a cell whose x2 coord is + # # supposed to be 160.137451171875 comes out as 160.13745498657227 because + # # of minus. :( + # vertical_uniq_locs = vertical_ruling_lines.map{|l| l.left.round(5)}.uniq #already sorted + # horizontal_uniq_locs = horizontal_ruling_lines.map{|l| l.top.round(5)}.uniq #already sorted - # call `rows` with `evaluate_cells` as `false` to defer filling in the text in - # each cell, which can be computationally intensive. - def rows(evaluate_cells=true) - if evaluate_cells - fill_in_cells! - end - tops = cells.map(&:top).uniq.sort - array_of_rows = tops.map do |top| - cells.select{|c| c.top == top }.sort_by(&:left) - end - #here, insert another kind of placeholder for empty corners - # like in 01001523B_China.pdf - #TODO: support placeholders for "empty" cells in rows other than row 1, and in #cols - # puts array_of_rows[0].inspect - if array_of_rows.size > 2 - if array_of_rows[0].map(&:left).uniq.size < array_of_rows[1].map(&:left).uniq.size - missing_spots = array_of_rows[1].map(&:left) - array_of_rows[0].map(&:left) + # cells.each do |c| + # vertical_rulings_spanned_over = vertical_uniq_locs.select{|l| l > c.left.round(5) && l < c.right.round(5) } + # horizontal_rulings_spanned_over = horizontal_uniq_locs.select{|t| t > c.top.round(5) && t < c.bottom.round(5) } - missing_spots.each do |missing_spot| - missing_spot_placeholder = Cell.new(array_of_rows[0][0].top, missing_spot, 0, 0) - missing_spot_placeholder.placeholder = true - array_of_rows[0] << missing_spot_placeholder - end - end - array_of_rows[0].sort_by!(&:left) - end - array_of_rows - end + # unless vertical_rulings_spanned_over.empty? + # c.spanning = true + # vertical_rulings_spanned_over.each do |spanned_over_line_loc| + # placeholder = Cell.new(c.top, spanned_over_line_loc, 0, c.height) + # placeholder.placeholder = true + # cells << placeholder + # end + # end + # unless horizontal_rulings_spanned_over.empty? + # c.spanning = true + # horizontal_rulings_spanned_over.each do |spanned_over_line_loc| + # placeholder = Cell.new(spanned_over_line_loc, c.left, c.width, 0) + # placeholder.placeholder = true + # cells << placeholder + # end + # end - # call `cols` with `evaluate_cells` as `false` to defer filling in the text in - # each cell, which can be computationally intensive. - def cols(evaluate_cells=true) - if evaluate_cells - fill_in_cells! - end - lefts = cells.map(&:left).uniq.sort - lefts.map do |left| - cells.select{|c| c.left == left }.sort_by(&:top) - end - end + # #if there's a spanning cell that's spans over both rows and columns, then it has "double placeholder" cells + # # e.g. ------------------- + # # | C | C | C | C | (this is some pretty sweet ASCII art, eh?) + # # |-----------------| + # # | C | C | C | C | + # # |-----------------| + # # | C | SC P | C | where MC is the "spanning cell" that holds all the text within its bounds + # # |---- + ----| P is a "placeholder" cell with either zero width or zero height + # # | C | P DP | C | DP is a "double placeholder" cell with zero width and zero height + # # |---- + ----| C is an ordinary cell. + # # | C | P DP | C | + # # |-----------------| - def to_a - fill_in_cells! - rows.map{ |row_cells| row_cells.map(&:text) } - end + # unless (double_placeholders = vertical_rulings_spanned_over.product(horizontal_rulings_spanned_over)).empty? + # double_placeholders.each do |vert_spanned_over, horiz_spanned_over| + # placeholder = Cell.new(horiz_spanned_over, vert_spanned_over, 0, 0) + # placeholder.placeholder = true + # cells << placeholder + # end + # end + # end + # end - def to_csv - out = StringIO.new - Tabula::Writers.CSV(rows, out) - out.string - end - - def to_tsv - out = StringIO.new - Tabula::Writers.TSV(rows, out) - out.string - end - - def to_json(*a) - { - 'json_class' => self.class.name, - 'extraction_method' => @extraction_method, - 'data' => rows, - }.to_json(*a) - end + def to_a + rows.map{ |row_cells| row_cells.map(&:text) } + end - def +(other) - raise ArgumentError unless other.page == @page - Spreadsheet.new(nil, nil, nil, nil, @page, @cells + other.cells, nil, nil ) - end + def +(other) + raise ArgumentError, "Data can only be added if it's from the same PDF page" unless other.page == @page + t = self.class.new(Java::TechnologyTabula::Utils.bounds(java.util.ArrayList.new([self, other])), + @page, + java.util.ArrayList.new(self.getCells + other.getCells), nil, nil) + t.setExtractionAlgorithm(Java::TechnologyTabulaExtractors::SpreadsheetExtractionAlgorithm.new) + t end end + +module Tabula + Spreadsheet = Java::TechnologyTabulaTableWithRulingLines +end diff --git a/lib/tabula/entities/table.rb b/lib/tabula/entities/table.rb index 01542d5..b7a4dc4 100644 --- a/lib/tabula/entities/table.rb +++ b/lib/tabula/entities/table.rb @@ -1,108 +1,65 @@ -module Tabula - class Table - attr_reader :extraction_method - attr_accessor :lines - def initialize(line_count, separators) - @separators = separators - @lines = (0...line_count).inject([]) { |m| m << Line.new } - @extraction_method = "original" - end - - def add_text_element(text_element, i, j) - if @lines.size <= i - @lines[i] = Line.new - end - if @lines[i].text_elements[j] - @lines[i].text_elements[j].merge!(text_element) - else - @lines[i].text_elements[j] = text_element - end - end - - def rpad! - max = lines.map{|l| l.text_elements.size}.max - lines.each do |line| - needed = max - line.text_elements.size - needed.times do - line.text_elements << TextElement.new(nil, nil, nil, nil, nil, nil, '', nil) - end - end - end - - def cols - rows.transpose - end +class Java::TechnologyTabula::Table + def to_csv + sb = java.lang.StringBuilder.new + Java::TechnologyTabulaWriters::CSVWriter.new.write(sb, self) + sb.toString + end - def rows - self.rpad! - lines.map do |l| - l.text_elements.map! do |te| - te || TextElement.new(nil, nil, nil, nil, nil, nil, '', nil) - end - end.sort_by { |l| l.map { |te| te.top || 0 }.max } - end + def to_tsv + sb = java.lang.StringBuilder.new + Java::TechnologyTabulaWriters::TSVWriter.new.write(sb, self) + sb.toString + end - # create a new Table object from an array of arrays, representing a list of rows in a spreadsheet - # probably only used for testing - def self.new_from_array(array_of_rows) - t = Table.new(array_of_rows.size, []) - @extraction_method = "testing" - array_of_rows.each_with_index do |row, index| - t.lines[index].text_elements = row.each_with_index.map{|cell, inner_index| TextElement.new(index, inner_index, 1, 1, nil, nil, cell, nil)} - end - t.rpad! - t - end + def to_json(*a) + sb = java.lang.StringBuilder.new + Java::TechnologyTabulaWriters::JSONWriter.new.write(sb, self) + sb.toString + end +end - #for equality testing, return @lines stripped of leading columns of empty strings - #TODO: write a method to strip all totally-empty columns (or not?) - def lstrip_lines - return @lines if @lines.include?(nil) - min_leading_empty_strings = Float::INFINITY - @lines.each do |line| - empties = line.text_elements.map{|t| t.nil? || t.text.empty? } - min_leading_empty_strings = [min_leading_empty_strings, empties.index(false)].min +class Tabula::Table < Java::TechnologyTabula::Table + # create a new Table object from an array of arrays, representing a list of rows in a spreadsheet + # probably only used for testing + def self.new_from_array(array_of_rows) + t = self.new + @extraction_method = "testing" + tlines = [] + array_of_rows.each_with_index do |row, i| + l = Tabula::Line.new + l.text_elements = row.each_with_index.map do |cell, j| + Tabula::TextElement.new(i.to_java(:float), j.to_java(:float), 1, 1, nil, 0, cell, 0) end - if min_leading_empty_strings == 0 - @lines - else - @lines.each{|line| line.text_elements = line.text_elements[min_leading_empty_strings..-1]} - @lines - end - end - def lstrip_lines! - @lines = self.lstrip_lines - end - - #used for testing, ignores separator locations (they'll sometimes be nil/empty) - def ==(other) - self.instance_variable_set(:@lines, self.lstrip_lines) - other.instance_variable_set(:@lines, other.lstrip_lines) - self.instance_variable_set(:@lines, self.lines.rpad(nil, other.lines.size)) - other.instance_variable_set(:@lines, other.lines.rpad(nil, self.lines.size)) - - self.lines.zip(other.lines).all? { |my, yours| my == yours } - + tlines << l end + t.instance_variable_set(:@lines, tlines) + t + end - def to_json(*a) - { - 'json_class' => self.class.name, - 'extraction_method' => @extraction_method, - 'data' => rows, - }.to_json(*a) - end + protected - def to_csv - out = StringIO.new - Tabula::Writers.CSV(rows, out) - out.string + #for equality testing, return @lines stripped of leading columns of empty strings + #TODO: write a method to strip all totally-empty columns (or not?) + def lstrip_lines + min_leading_empty_strings = ::Float::INFINITY + lines.each do |line| + empties = line.text_elements.map{|t| t.nil? || t.text.empty? } + min_leading_empty_strings = [min_leading_empty_strings, + empties.index(false) || 0].min end - - def to_tsv - out = StringIO.new - Tabula::Writers.TSV(rows, out) - out.string + if min_leading_empty_strings == 0 + lines + else + (0...lines.size).each do |i| + lines[i].text_elements.removeRange(0, min_leading_empty_strings) + end + lines end end + def lstrip_lines! + #@lines = self.lstrip_lines + lstrip_lines + end + + attr_accessor :lines end diff --git a/lib/tabula/entities/text_chunk.rb b/lib/tabula/entities/text_chunk.rb index 41016ca..f995543 100644 --- a/lib/tabula/entities/text_chunk.rb +++ b/lib/tabula/entities/text_chunk.rb @@ -1,162 +1,20 @@ -module Tabula - ## - # a "collection" of TextElements - class TextChunk < ZoneEntity - attr_accessor :font, :font_size, :text_elements, :width_of_space - - SPACE_RUN_MAX_LENGTH = 3 - - ## - # initialize a new TextChunk from a TextElement - def self.create_from_text_element(text_element) - raise TypeError, "argument is not a TextElement" unless text_element.instance_of?(TextElement) - tc = self.new(text_element.top, text_element.left, text_element.width, text_element.height) - tc.text_elements = [text_element] - return tc - end - - ## - # group an iterable of TextChunk into a list of Line - def self.group_by_lines(text_chunks) - lines = [] - text_chunks.each do |te| - next if te.text =~ ONLY_SPACES_RE - l = lines.find { |line| line.horizontal_overlap_ratio(te) >= 0.01 } - if l.nil? - l = Line.new - lines << l - end - l << te - end - - # for each line, remove runs of the space char - # should not change dimensions of the container +Line+ - lines.each do |l| - l.text_elements = l.text_elements.reduce([]) do |memo, text_chunk| - long_space_runs = text_chunk - .text_elements - .chunk { |te| te.text == ' '} # detect runs of spaces... - .select { |is_space, text_elements| # ...longer than SPACE_RUN_MAX_LENGTH - is_space && !text_elements.nil? && text_elements.size >= SPACE_RUN_MAX_LENGTH - } - .map { |_, text_elements| text_elements } - - # no long runs of spaces - # keep as it was and end iteration - if long_space_runs.empty? - memo << text_chunk - next memo - end - - ranges = long_space_runs.map { |lsr| - idx = text_chunk - .text_elements - .index { |te| te.equal?(lsr.first) } # we need pointer comparison here - (idx)..(idx+lsr.size-1) - } - - in_run = false - new_chunk = true - text_chunk - .text_elements - .each_with_index do |te, i| - if ranges.any? { |r| r.include?(i) } # te belongs to a run of spaces, skip - in_run = true - else - if in_run || new_chunk - memo << TextChunk.create_from_text_element(te) - else - memo.last << te - end - in_run = new_chunk = false - end - end - memo - end # reduce - end # each - lines - end +java_import Java::TechnologyTabula::TextChunk - ## - # calculate estimated columns from an iterable of +Tabula::Line+ - def self.column_positions(lines) - right = 0 - columns = [] +## +# a "collection" of TextElements +# class TextChunk +# attr_accessor :font, :font_size, :width_of_space - top = lines.min_by(&:top).top +# def inspect +# "#" +# end - lines.map(&:text_elements).flatten.each do |te| - next if te.text =~ ONLY_SPACES_RE - if te.top >= top - left = te.left - if (left > right) - columns << right - right = te.right - elsif te.right > right - right = te.right - end - end - end - columns - end +# def to_h +# super.merge(:text => self.text) +# end +# end - ## - # add a TextElement to this TextChunk - def <<(text_element) - self.text_elements << text_element - self.merge!(text_element) - end - def merge!(other) - if other.instance_of?(TextChunk) - if self.horizontally_overlaps?(other) && other.top < self.top - self.text_elements = other.text_elements + self.text_elements - else - self.text_elements = self.text_elements + other.text_elements - end - end - super(other) - end - - ## - # split this TextChunk vertically - # (in place, returns the remaining chunk) - def split_vertically!(y) - raise "Not Implemented" - end - - ## - # remove leading and trailing whitespace - # (changes geometry accordingly) - # TODO horrible implementation - fix. - def strip! - acc = 0 - new_te = self.text_elements.drop_while { |te| - te.text == ' ' && acc += 1 - } - self.left += self.text_elements.take(acc).inject(0) { |m, te| m += te.width } - self.text_elements = new_te - - self.text_elements.reverse! - acc = 0 - new_te = self.text_elements.drop_while { |te| - te.text == ' ' && acc += 1 - } - self.right -= self.text_elements.take(acc).inject(0) { |m, te| m += te.width } - self.text_elements = new_te.reverse - self - end - - def text - self.text_elements.map(&:text).join - end - - def inspect - "#" - end - - def to_h - super.merge(:text => self.text) - end - end +module Tabula + TextChunk = Java::TechnologyTabula::TextChunk end diff --git a/lib/tabula/entities/text_element.rb b/lib/tabula/entities/text_element.rb index 3f463b4..90c4fe0 100644 --- a/lib/tabula/entities/text_element.rb +++ b/lib/tabula/entities/text_element.rb @@ -1,111 +1,10 @@ module Tabula ## # a Glyph - class TextElement < ZoneEntity - attr_accessor :font, :font_size, :text, :width_of_space, :direction + #TextElement = Java::TechnologyTabula::TextElement + class TextElement < Java::TechnologyTabula::TextElement - TOLERANCE_FACTOR = 0.25 - - def initialize(top, left, width, height, font, font_size, text, width_of_space, direction=0) - super(top, left, width, height) - self.font = font - self.font_size = font_size - self.text = text - self.width_of_space = width_of_space - self.direction = direction - end - - EMPTY = TextElement.new(0, 0, 0, 0, nil, 0, '', 0) - - ## - # heuristically merge an iterable of TextElement into a list of TextChunk - def self.merge_words(text_elements, options={}) - default_options = {:vertical_rulings => []} - options = default_options.merge(options) - vertical_ruling_locations = options[:vertical_rulings].map(&:left) if options[:vertical_rulings] - - return [] if text_elements.empty? - - text_chunks = [TextChunk.create_from_text_element(text_elements.shift)] - - text_elements.inject(text_chunks) do |chunks, char| - current_chunk = chunks.last - prev_char = current_chunk.text_elements.last - - # if same char AND overlapped, skip - if prev_char.text == char.text && prev_char.overlaps_with_ratio?(char, 0.85) - chunks - else - # any vertical ruling goes across prev_char and char? - across_vertical_ruling = vertical_ruling_locations.any? { |loc| - prev_char.left < loc && char.left > loc - } - - # should we add a space? - if (prev_char.text != " ") && (char.text != " ") \ - && !across_vertical_ruling \ - && prev_char.should_add_space?(char) - - sp = self.new(prev_char.top, - prev_char.right, - prev_char.width_of_space, - prev_char.width_of_space, # width == height for spaces - prev_char.font, - prev_char.font_size, - ' ', - prev_char.width_of_space) - chunks.last << sp - prev_char = sp - end - - # should_merge? isn't aware of vertical rulings, so even if two text elements are close enough - # that they ought to be merged by that account. - # we still shouldn't merge them if the two elements are on opposite sides of a vertical ruling. - # Why are both of those `.left`?, you might ask. The intuition is that a letter - # that starts on the left of a vertical ruling ought to remain on the left of it. - if !across_vertical_ruling && prev_char.should_merge?(char) - chunks.last << char - else - # create a new chunk - chunks << TextChunk.create_from_text_element(char) - end - chunks - end - end - end - - # more or less returns True if distance < tolerance - def should_merge?(other) - raise TypeError, "argument is not a TextElement" unless other.instance_of?(TextElement) - self.vertically_overlaps?(other) && self.horizontal_distance(other) < width_of_space * (1 + TOLERANCE_FACTOR) && !self.should_add_space?(other) - end - - # more or less returns True if (tolerance <= distance < CHARACTER_DISTANCE_THRESHOLD*tolerance) - def should_add_space?(other) - raise TypeError, "argument is not a TextElement" unless other.instance_of?(TextElement) - - return false if self.width_of_space.nan? - - (self.vertically_overlaps?(other) && - self.horizontal_distance(other).abs.between?(self.width_of_space * (1 - TOLERANCE_FACTOR), self.width_of_space * (1 + TOLERANCE_FACTOR))) || - (self.vertical_distance(other) > self.height) - end - - ## - # merge this TextElement with another (adjust size and text content accordingly) - def merge!(other) - raise TypeError, "argument is not a TextElement" unless other.instance_of?(TextElement) - if self.horizontally_overlaps?(other) and other.top < self.top - self.text = other.text + self.text - else - self.text << other.text - end - super(other) - end - - def to_h - super.merge({:font => self.font, :text => self.text }) - end + EMPTY = TextElement.new(0, 0, 0, 0, nil, 0, '', 0, 0) def inspect "#" @@ -114,17 +13,6 @@ def inspect def ==(other) self.text.strip == other.text.strip end - - # sort in lexicographic (reading) order - def <=>(other) - if self.vertically_overlaps?(other) - self.left <=> other.left - elsif self.top < other.top - -1 - else - 1 - end - end - end + end diff --git a/lib/tabula/entities/text_element_index.rb b/lib/tabula/entities/text_element_index.rb deleted file mode 100644 index 64ec8b1..0000000 --- a/lib/tabula/entities/text_element_index.rb +++ /dev/null @@ -1,55 +0,0 @@ -module Tabula - class TextElementIndex < Java::ComInfomatiqJsiRtree::RTree - - attr_reader :te_dict - - class SaveToListProcedure - include Java::GnuTroveProcedure::TIntProcedure - - attr_reader :list - - def initialize(parent) - @parent = parent - @list = [] - end - - def execute(id) - @list << @parent.te_dict[id] - return true - end - - def reset! - @list = [] - end - - end - - def initialize - super - self.init(nil) - @te_dict = {} - @save_to_list = SaveToListProcedure.new(self) - end - - def <<(text_element) - r = Java::ComInfomatiqJsi::Rectangle.new(text_element.left, - text_element.top, - text_element.right, - text_element.bottom) - @te_dict[text_element.object_id] = text_element - self.add(r, text_element.object_id) - end - - def contains(zone_entity) - r = Java::ComInfomatiqJsi::Rectangle.new(zone_entity.left, - zone_entity.top, - zone_entity.right, - zone_entity.bottom) - @save_to_list.reset! - super(r, @save_to_list) - - # sort in lexicographic (reading) order - @save_to_list.list.sort - end - end -end diff --git a/lib/tabula/entities/zone_entity.rb b/lib/tabula/entities/zone_entity.rb deleted file mode 100644 index dca6543..0000000 --- a/lib/tabula/entities/zone_entity.rb +++ /dev/null @@ -1,57 +0,0 @@ -java_import java.awt.geom.Point2D - -module Tabula - - class ZoneEntity < java.awt.geom.Rectangle2D::Float - - attr_accessor :texts - - def initialize(top, left, width, height) - super() - if left && top && width && height - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], left, top, width, height - end - self.texts = [] - end - - def merge!(other) - self.top = [self.top, other.top].min - self.left = [self.left, other.left].min - self.width = [self.right, other.right].max - left - self.height = [self.bottom, other.bottom].max - top - - self.java_send :setRect, [Java::float, Java::float, Java::float, Java::float,], self.left, self.top, self.width, self.height - end - - ## - # default sorting order for ZoneEntity objects - # is lexicographical (left to right, top to bottom) - def <=>(other) - return 1 if self.left > other.left - return -1 if self.left < other.left - return 0 if self.vertically_overlaps?(other) - return 1 if self.top > other.top - return -1 if self.top < other.top - return 0 - end - - def to_json(options={}) - self.to_h.to_json - end - - def inspect - "#<#{self.class} dims: #{self.dims(:top, :left, :width, :height)}>" - end - - def tlbr - [top, left, bottom, right] - end - - def points - [ Point2D::Float.new(left, top), - Point2D::Float.new(right, top), - Point2D::Float.new(right, bottom), - Point2D::Float.new(left, bottom) ] - end - end -end diff --git a/lib/tabula/extraction.rb b/lib/tabula/extraction.rb index 066440c..d5cbf3c 100644 --- a/lib/tabula/extraction.rb +++ b/lib/tabula/extraction.rb @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- -java_import org.apache.pdfbox.pdfparser.PDFParser -java_import org.apache.pdfbox.util.TextPosition java_import org.apache.pdfbox.pdmodel.PDDocument -java_import org.apache.pdfbox.util.PDFTextStripper java_import org.apache.pdfbox.pdmodel.encryption.StandardDecryptionMaterial -java_import java.awt.geom.AffineTransform module Tabula @@ -20,317 +16,45 @@ def Extraction.openPDF(pdf_filename, password='') document end - class ObjectExtractor < org.apache.pdfbox.pdfviewer.PageDrawer - - attr_accessor :characters, :debug_text, :debug_clipping_paths, :clipping_paths, :options - field_accessor :pageSize, :page - - PRINTABLE_RE = /[[:print:]]/ - DEFAULT_OPTIONS = { - :line_color_filter => nil, - :extract_ruling_lines => true - } - - def initialize(pdf_filename, pages=[1], password='', options={}) - raise Errno::ENOENT unless File.exists?(pdf_filename) + class PagesInfoExtractor + def initialize(pdf_filename, password='') @pdf_filename = pdf_filename @pdf_file = Extraction.openPDF(pdf_filename, password) @all_pages = @pdf_file.getDocumentCatalog.getAllPages - @pages = pages == :all ? (1..@all_pages.size) : pages - - super() - - self.options = DEFAULT_OPTIONS.merge(options) - self.characters = [] - @debug_clipping_paths = false - @clipping_path = nil - @transformed_clipping_path = nil - self.clipping_paths = [] - @rulings = [] - @min_char_width = @min_char_height = 1000000 end - def extract + def pages Enumerator.new do |y| begin - @pages.each do |i| - if i-1 >= @all_pages.size || (i-1) < 0 - raise IndexError, "Page #{i} doesn't exist. Skipping. Valid pages are 1..#{@all_pages.size}" - end - page = @all_pages.get(i-1) + @all_pages.each_with_index do |page, i| contents = page.getContents - next if contents.nil? - self.clear! - self.drawPage(page) - p = Tabula::Page.new(@pdf_filename, - page.findCropBox.width, - page.findCropBox.height, - page.getRotation.to_i, - i, #one-indexed, just like `i` is. - self.characters, - self.rulings, - @min_char_width, - @min_char_height) - y.yield p + y.yield Tabula::Page.new(0.to_java(:float), + 0.to_java(:float), + page.findCropBox.width.to_java(:float), + page.findCropBox.height.to_java(:float), + page.getRotation.to_i.to_java(:int), + (i+1).to_java(:int)) # remember, these are one-indexed end ensure @pdf_file.close - end # begin - end - end - - def clear! - self.characters.clear - self.clipping_paths.clear - @page_transform = nil - @rulings.clear - end - - def ensurePageSize! - if self.pageSize.nil? && !self.page.nil? - mediaBox = self.page.findMediaBox - self.pageSize = (mediaBox == nil ? nil : mediaBox.createDimension) - end - end - - def drawPage(page) - self.page = page - if !self.page.getContents.nil? - ensurePageSize! - self.processStream(self.page, - self.page.findResources, - self.page.getContents.getStream) - end - end - - def setStroke(stroke) - @basicStroke = stroke - end - - def getStroke - @basicStroke - end - - - def strokePath(filter_by_color=nil) - unless self.options[:extract_ruling_lines] - self.getLinePath.reset - return - end - - path = self.pathToList(self.getLinePath) - - if path[0][0] != java.awt.geom.PathIterator::SEG_MOVETO \ - || path[1..-1].any? { |p| p.first != java.awt.geom.PathIterator::SEG_LINETO && p.first != java.awt.geom.PathIterator::SEG_MOVETO && p.first != java.awt.geom.PathIterator::SEG_CLOSE } - self.getLinePath.reset - return - end - - ccp_bounds = self.currentClippingPath - - strokeColorComps = filter_by_color || self.getGraphicsState.getStrokingColor.getJavaColor.getRGBColorComponents(nil) - color_filter = self.options[:line_color_filter] - - first = path.shift - start_pos = java.awt.geom.Point2D::Float.new(first[1][0], first[1][1]) - - path.each do |p| - end_pos = java.awt.geom.Point2D::Float.new(p[1][0], p[1][1]) - line = (start_pos <=> end_pos) == -1 \ - ? java.awt.geom.Line2D::Float.new(start_pos, end_pos) \ - : java.awt.geom.Line2D::Float.new(end_pos, start_pos) - - if p[0] == java.awt.geom.PathIterator::SEG_LINETO \ - && (color_filter.nil? ? true : color_filter.call(strokeColorComps)) \ - && line.intersects(ccp_bounds) - # convert line to rectangle for clipping it to the current clippath - # sucks, but awt doesn't have methods for this - tmp = line.getBounds2D.createIntersection(ccp_bounds).getBounds2D - @rulings << ::Tabula::Ruling.new(tmp.getY, - tmp.getX, - tmp.getWidth, - tmp.getHeight, - filter_by_color.to_a) end - start_pos = end_pos end - self.getLinePath.reset - end - - def fillPath(windingRule) - self.strokePath(self.getGraphicsState.getNonStrokingColor.getJavaColor.getRGBColorComponents(nil)) end - - def drawImage(image, at) - end - - def transformPath(path) - self.pageTransform.createTransformedShape(path) - end - - def pageTransform - unless @page_transform.nil? - return @page_transform - end - - cb = page.findCropBox - if !([90, -270, -90, 270].include?(page.getRotation)) - @page_transform = AffineTransform.getScaleInstance(1, -1) - @page_transform.translate(0, -cb.getHeight) - else - @page_transform = AffineTransform.getScaleInstance(-1, 1) - @page_transform.rotate(page.getRotation * (Math::PI/180.0), - cb.getLowerLeftX, cb.getLowerLeftY) - end - @page_transform - end - - def currentClippingPath - cp = self.getGraphicsState.getCurrentClippingPath - - if cp == @clipping_path - return @transformed_clipping_path_bounds - end - - @clipping_path = cp - @transformed_clipping_path = self.transformPath(cp) - @transformed_clipping_path_bounds = @transformed_clipping_path.getBounds - - return @transformed_clipping_path_bounds - end - - def processTextPosition(text) - c = text.getCharacter - h = text.getHeightDir.round(2) - - if c == ' ' || c == ' ' # replace non-breaking space for space - c = ' ' - h = text.getWidth.round(2) - end - - te = Tabula::TextElement.new(text.getY.round(2) - h, - text.getX.round(2), - text.getWidth.round(2), - # ugly hack follows: we need spaces to have a height, so we can - # test for vertical overlap. height == width seems a safe bet. - h, - text.getFont, - text.getFontSize.round(2), - c, - # workaround a possible bug in PDFBox: https://issues.apache.org/jira/browse/PDFBOX-1755 - text.getWidthOfSpace == 0 ? self.currentSpaceWidth : text.getWidthOfSpace, - text.getDir) - - ccp_bounds = self.currentClippingPath - - if self.debug_clipping_paths && !self.clipping_paths.include?(ccp_bounds) - self.clipping_paths << ::Tabula::ZoneEntity.new(ccp_bounds.getMinY, - ccp_bounds.getMinX, - ccp_bounds.getWidth, - ccp_bounds.getHeight) - end - - if te.width < @min_char_width - @min_char_width = te.width - end - - if te.height < @min_char_height - @min_char_height = te.height - end - - if c =~ PRINTABLE_RE && ccp_bounds.intersects(te) - self.characters << te - end - end - - def page_count - @all_pages.size - end - - def rulings - return [] if @rulings.empty? - @rulings.reject { |l| (l.left == l.right && l.top == l.bottom) || [l.top, l.left, l.bottom, l.right].any? { |p| p < 0 } } - end - - protected - - # workaround a possible bug in PDFBox: https://issues.apache.org/jira/browse/PDFBOX-1755 - def currentSpaceWidth - gs = self.getGraphicsState - font = gs.getTextState.getFont - - fontSizeText = gs.getTextState.getFontSize - horizontalScalingText = gs.getTextState.getHorizontalScalingPercent / 100.0 - - if font.java_kind_of?(org.apache.pdfbox.pdmodel.font.PDType3Font) - puts "TYPE3" - end - - # idea from pdf.js - # https://github.com/mozilla/pdf.js/blob/master/src/core/fonts.js#L4418 - spaceWidthText = spaceWidthText = [' ', '-', '1', 'i'] \ - .map { |c| font.getFontWidth(c.ord) } \ - .find { |w| w > 0 } || 1000 - - ctm00 = gs.getCurrentTransformationMatrix.getValue(0, 0) - - return (spaceWidthText/1000.0) * fontSizeText * horizontalScalingText * (ctm00 == 0 ? 1 : ctm00) - end - - def pathToList(path) - iterator = path.getPathIterator(self.pageTransform) - rv = [] - while !iterator.isDone do - coords = Java::double[6].new - segType = iterator.currentSegment(coords) - rv << [segType, coords] - iterator.next - end - rv - end - - def debugPath(path) - rv = '' - pathToList(path).each do |segType, coords| - case segType - when java.awt.geom.PathIterator::SEG_MOVETO - rv += "MOVE: #{coords[0]} #{coords[1]}\n" - when java.awt.geom.PathIterator::SEG_LINETO - rv += "LINE: #{coords[0]} #{coords[1]}\n" - when java.awt.geom.PathIterator::SEG_CLOSE - rv += "CLOSE\n\n" - end - end - rv - end - end + class ObjectExtractor < Java::TechnologyTabula::ObjectExtractor - class PagesInfoExtractor - def initialize(pdf_filename, password='') - @pdf_filename = pdf_filename - @pdf_file = Extraction.openPDF(pdf_filename, password) - @all_pages = @pdf_file.getDocumentCatalog.getAllPages - end + alias_method :close!, :close - def pages - Enumerator.new do |y| - begin - @all_pages.each_with_index do |page, i| - contents = page.getContents + # TODO: the +pages+ constructor argument does not make sense + # now that we have +extract_page+ and +extract_pages+ + def initialize(pdf_filename, pages=[1], password='', options={}) + raise Errno::ENOENT unless File.exists?(pdf_filename) + @pdf_filename = pdf_filename + document = Extraction.openPDF(pdf_filename, password) - y.yield Tabula::Page.new(@pdf_filename, - page.findCropBox.width, - page.findCropBox.height, - page.getRotation.to_i, - i+1) #remember, these are one-indexed - end - ensure - @pdf_file.close - end - end + super(document) end end end diff --git a/lib/tabula/line_segment_detector.rb b/lib/tabula/line_segment_detector.rb deleted file mode 100644 index 44302ae..0000000 --- a/lib/tabula/line_segment_detector.rb +++ /dev/null @@ -1,130 +0,0 @@ -require 'java' -require 'rbconfig' - -require 'ffi' - -require_relative './entities' -require_relative './pdf_render' -require_relative './extraction' - -java_import javax.imageio.ImageIO -java_import java.awt.image.BufferedImage -java_import org.apache.pdfbox.pdmodel.PDDocument - -module Tabula - module LSD - extend FFI::Library - ffi_lib File.expand_path('../../ext/' + case RbConfig::CONFIG['host_os'] - when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ - if RbConfig::CONFIG['host_cpu'] == 'x86_64' - 'liblsd64.dll' - else - 'liblsd.dll' - end - when /darwin|mac os/ - 'liblsd.dylib' - when /linux/ - if RbConfig::CONFIG['target_cpu'] == 'x86_64' - 'liblsd-linux64.so' - else - 'liblsd-linux32.so' - end - else - raise "unknown os: #{RbConfig::CONFIG['host_os']}" - end, - File.dirname(__FILE__)) - - attach_function :lsd, [ :pointer, :buffer_in, :int, :int ], :pointer - attach_function :free_values, [ :pointer ], :void - - DETECT_LINES_DEFAULTS = { - :scale_factor => nil, - :image_size => 2048 - } - - def LSD.detect_lines_in_pdf(pdf_path, options={}) - options = DETECT_LINES_DEFAULTS.merge(options) - - pdf_file = PDDocument.loadNonSeq(java.io.File.new(pdf_path), nil) - lines = pdf_file.getDocumentCatalog.getAllPages.to_a.map do |page| - bi = Tabula::Render.pageToBufferedImage(page, options[:image_size]) - detect_lines(bi, options[:scale_factor] || (page.findCropBox.width / options[:image_size])) - end - pdf_file.close - lines - end - - #zero-indexed page_number - def LSD.detect_lines_in_pdf_page(pdf_path, page_number, options={}) - options = DETECT_LINES_DEFAULTS.merge(options) - - pdf_file = Extraction.openPDF(pdf_path) - page = pdf_file.getDocumentCatalog.getAllPages[page_number] - bi = Tabula::Render.pageToBufferedImage(page, - options[:image_size]) - pdf_file.close - detect_lines(bi, - options[:scale_factor] || (page.findCropBox.width / options[:image_size])) - end - - # image can be either a string (path to image) or a Java::JavaAwtImage::BufferedImage - # image to pixels: http://stackoverflow.com/questions/6524196/java-get-pixel-array-from-image - def LSD.detect_lines(image, scale_factor=1) - - bimage = if image.class == Java::JavaAwtImage::BufferedImage - image - elsif image.class == String - ImageIO.read(java.io.File.new(image)) - else - raise ArgumentError, 'image must be a string or a BufferedImage' - end - - image = LSD.image_to_image_float(bimage) - - lines_found_ptr = FFI::MemoryPointer.new(:int, 1) - - out = lsd(lines_found_ptr, image, bimage.getWidth, bimage.getHeight) - - lines_found = lines_found_ptr.get_int - - rv = [] - lines_found.times do |i| - a = out[7*4*i].read_array_of_type(:float, 7) - - a_round = a[0..3].map(&:round) - p1, p2 = [[a_round[0], a_round[1]], [a_round[2], a_round[3]]] - - rv << Tabula::Ruling.new(p1[1] * scale_factor, - p1[0] * scale_factor, - (p2[0] - p1[0]) * scale_factor, - (p2[1] - p1[1]) * scale_factor) - end - - free_values(out) - bimage.flush - bimage.getGraphics.dispose - image = nil - - return rv - end - - private - - def LSD.image_to_image_float(buffered_image) - width = buffered_image.getWidth; height = buffered_image.getHeight - raster_size = width * height - - image_float = FFI::MemoryPointer.new(:float, raster_size) - pixels = Java::int[width * height].new - buffered_image.getRGB(0, 0, width, height, pixels, 0, width) - - image_float.put_array_of_float 0, pixels.to_a - end - - - end -end - -if __FILE__ == $0 - puts Tabula::LSD.detect_lines_in_pdf_page ARGV[0], ARGV[1].to_i -end diff --git a/lib/tabula/pdf_line_extractor.rb b/lib/tabula/pdf_line_extractor.rb deleted file mode 100644 index 03121ff..0000000 --- a/lib/tabula/pdf_line_extractor.rb +++ /dev/null @@ -1,319 +0,0 @@ -java_import org.apache.pdfbox.util.operator.OperatorProcessor -java_import org.apache.pdfbox.pdfparser.PDFParser -java_import org.apache.pdfbox.util.PDFStreamEngine -java_import org.apache.pdfbox.util.ResourceLoader - -java_import java.awt.geom.PathIterator -java_import java.awt.geom.Point2D -java_import java.awt.geom.GeneralPath -java_import java.awt.geom.AffineTransform -java_import java.awt.Color - -warn 'Tabula::Extraction::LineExtractor is DEPRECATED and will be removed' - -class Tabula::Extraction::LineExtractor < org.apache.pdfbox.util.PDFStreamEngine - - attr_accessor :currentX, :currentY - attr_accessor :currentPath - attr_accessor :rulings - attr_accessor :options - field_accessor :page - - DETECT_LINES_DEFAULTS = { - :snapping_grid_cell_size => 2 - } - - def self.collapse_vertical_rulings(lines) #lines should all be of one orientation (i.e. horizontal, vertical) - lines.sort!{|a, b| a.left != b.left ? a.left <=> b.left : a.top <=> b.top } - lines.inject([]) do |memo, next_line| - if memo.last && next_line.left == memo.last.left && memo.last.nearlyIntersects?(next_line) - memo.last.top = [next_line.top, memo.last.top].min - memo.last.bottom = [next_line.bottom, memo.last.bottom].max - memo - else - memo << next_line - end - end - end - - def self.collapse_horizontal_rulings(lines) #lines should all be of one orientation (i.e. horizontal, vertical) - lines.sort!{|a, b| a.top != b.top ? a.top <=> b.top : a.left <=> b.left } - lines.inject([]) do |memo, next_line| - if memo.last && next_line.top == memo.last.top && memo.last.nearlyIntersects?(next_line) - memo.last.left = [next_line.left, memo.last.left].min - memo.last.right = [next_line.right, memo.last.right].max - memo - else - memo << next_line - end - end - end - - #N.B. for merge `spreadsheets` into `text-extractor-refactor` -- - # only substantive change here is calling Tabula::Ruling::clean_rulings on LSD output in this method - # the rest is readability changes. - #page_number here is zero-indexed - def self.lines_in_pdf_page(pdf_path, page_number, options={}) - options = options.merge!(DETECT_LINES_DEFAULTS) - if options[:render_pdf] - # only LSD rulings need to be "cleaned" with clean_rulings; might as well do this here - # since there's no good reason want unclean lines - Tabula::Ruling::clean_rulings(Tabula::LSD::detect_lines_in_pdf_page(pdf_path, page_number, options)) - else - pdf_file = ::Tabula::Extraction.openPDF(pdf_path) - page = pdf_file.getDocumentCatalog.getAllPages[page_number] - le = self.new(options) - le.processStream(page, page.findResources, page.getContents.getStream) - pdf_file.close - rulings = le.rulings.map do |l, color| - ::Tabula::Ruling.new(l.getP1.getY, - l.getP1.getX, - l.getP2.getX - l.getP1.getX, - l.getP2.getY - l.getP1.getY, - color) - end - rulings.reject! { |l| (l.left == l.right && l.top == l.bottom) || [l.top, l.left, l.bottom, l.right].any? { |p| p < 0 } } - collapse_vertical_rulings(rulings.select(&:vertical?)) + collapse_horizontal_rulings(rulings.select(&:horizontal?)) - end - end - - class LineToOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - x, y = arguments[0], arguments[1] - ppos = drawer.TransformedPoint(x.floatValue, y.floatValue) - - l = java.awt.geom.Line2D::Float.new(drawer.currentX, drawer.currentY, ppos.getX, ppos.getY) - - drawer.currentPath << l if l.horizontal? or l.vertical? - - drawer.currentX, drawer.currentY = ppos.getX, ppos.getY - end - end - - class MoveToOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - x, y = arguments[0], arguments[1] - - ppos = drawer.TransformedPoint(x.floatValue, y.floatValue) - - drawer.currentX, drawer.currentY = ppos.getX, ppos.getY - end - end - - class AppendRectangleToPathOperator < OperatorProcessor - def process(operator, arguments) - - drawer = self.context - finalX, finalY, finalW, finalH = arguments.to_array.map(&:floatValue) - - ppos = drawer.TransformedPoint(finalX, finalY) - psize = drawer.ScaledPoint(finalW, finalH) - - finalY = ppos.getY - psize.getY - if finalY < 0 - finalY = 0 - end - - width = psize.getX.abs - height = psize.getY.abs - - lines = if width > height && height < 2 # horizontal line, "thin" rectangle. - [java.awt.geom.Line2D::Float.new(ppos.getX, finalY + psize.getY/2, ppos.getX + psize.getX, finalY + psize.getY/2)] - elsif width < height && width < 2 # vertical line, "thin" rectangle - [java.awt.geom.Line2D::Float.new(ppos.getX + psize.getX/2, finalY, ppos.getX + psize.getX/2, finalY + psize.getY)] - else - # add every edge of the rectangle to drawer.rulings - [java.awt.geom.Line2D::Float.new(ppos.getX, finalY, ppos.getX + psize.getX, finalY), - java.awt.geom.Line2D::Float.new(ppos.getX, finalY, ppos.getX, finalY + psize.getY), - java.awt.geom.Line2D::Float.new(ppos.getX+psize.getX, finalY, ppos.getX + psize.getX, finalY + psize.getY), - java.awt.geom.Line2D::Float.new(ppos.getX, finalY+psize.getY, ppos.getX + psize.getX, finalY + psize.getY)] - end - - drawer.currentPath += lines.select { |l| l.horizontal? or l.vertical? } - - end - end - - class StrokePathOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - strokeColorComps = drawer.getGraphicsState.getStrokingColor.getJavaColor.getRGBColorComponents(nil) - color_filter = drawer.options[:line_color_filter] || lambda{|c| true } #by default, use all lines, regardless of color - if color_filter.call(strokeColorComps) - drawer.currentPath.each { |segment| drawer.addRuling(segment, strokeColorComps.to_a) } - end - - drawer.currentPath = [] - end - end - - class CloseFillNonZeroAndStrokePathOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - - fillColorComps = drawer.getGraphicsState.getNonStrokingColor.getJavaColor.getRGBColorComponents(nil) - color_filter = drawer.options[:line_color_filter] || lambda{|c| true } #by default, use all lines, regardless of color - if color_filter.call(fillColorComps) - drawer.currentPath.each { |segment| drawer.addRuling(segment, fillColorComps.to_a) } - end - - drawer.currentPath = [] - end - end - - class CloseAndStrokePathOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - drawer.currentPath.each { |segment| drawer.addRuling(segment) } - drawer.currentPath = [] - end - end - - class EndPathOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - # end without stroke, we don't care about it. discard it - drawer.currentPath = [] - end - end - - class FillNonZeroRuleOperator < OperatorProcessor - def process(operator, arguments) - drawer = self.context - # end without stroke, we don't care about it. discard it - drawer.currentPath = [] - end - end - - OPERATOR_PROCESSORS = { - 'm' => MoveToOperator.new, - 're' => AppendRectangleToPathOperator.new, - 'l' => LineToOperator.new, - 'S' => StrokePathOperator.new, - 's' => StrokePathOperator.new, - 'n' => EndPathOperator.new, - 'b' => CloseFillNonZeroAndStrokePathOperator.new, - 'b*' => CloseFillNonZeroAndStrokePathOperator.new, - 'f' => CloseFillNonZeroAndStrokePathOperator.new, - 'f*' => CloseFillNonZeroAndStrokePathOperator.new, - 'BT' => org.apache.pdfbox.util.operator.BeginText.new, - 'cm' => org.apache.pdfbox.util.operator.Concatenate.new, - 'CS' => org.apache.pdfbox.util.operator.SetStrokingColorSpace.new, - 'cs' => org.apache.pdfbox.util.operator.SetNonStrokingColorSpace.new, - 'ET' => org.apache.pdfbox.util.operator.EndText.new, - 'G' => org.apache.pdfbox.util.operator.SetStrokingGrayColor.new, - 'g' => org.apache.pdfbox.util.operator.SetNonStrokingGrayColor.new, - 'gs' => org.apache.pdfbox.util.operator.SetGraphicsStateParameters.new, - 'K' => org.apache.pdfbox.util.operator.SetStrokingCMYKColor.new, - 'k' => org.apache.pdfbox.util.operator.SetNonStrokingCMYKColor.new, - 'q' => org.apache.pdfbox.util.operator.GSave.new, - 'Q' => org.apache.pdfbox.util.operator.GRestore.new, - 'RG' => org.apache.pdfbox.util.operator.SetStrokingRGBColor.new, - 'rg' => org.apache.pdfbox.util.operator.SetNonStrokingRGBColor.new, - 'SC' => org.apache.pdfbox.util.operator.SetStrokingColor.new, - 'sc' => org.apache.pdfbox.util.operator.SetNonStrokingColor.new, - 'SCN' => org.apache.pdfbox.util.operator.SetStrokingColor.new, - 'scn' => org.apache.pdfbox.util.operator.SetNonStrokingColor.new, - 'T*' => org.apache.pdfbox.util.operator.NextLine.new, - 'Tc' => org.apache.pdfbox.util.operator.SetCharSpacing.new, - 'Td' => org.apache.pdfbox.util.operator.MoveText.new, - 'TD' => org.apache.pdfbox.util.operator.MoveTextSetLeading.new, - 'Tf' => org.apache.pdfbox.util.operator.SetTextFont.new, - 'Tj' => org.apache.pdfbox.util.operator.ShowText.new, - 'TJ' => org.apache.pdfbox.util.operator.ShowTextGlyph.new, - 'TL' => org.apache.pdfbox.util.operator.SetTextLeading.new, - 'Tm' => org.apache.pdfbox.util.operator.SetMatrix.new, - 'Tr' => org.apache.pdfbox.util.operator.SetTextRenderingMode.new, - 'Ts' => org.apache.pdfbox.util.operator.SetTextRise.new, - 'Tw' => org.apache.pdfbox.util.operator.SetWordSpacing.new, - 'Tz' => org.apache.pdfbox.util.operator.SetHorizontalTextScaling.new, - "\'" => org.apache.pdfbox.util.operator.MoveAndShow.new, - '\"' => org.apache.pdfbox.util.operator.SetMoveAndShow.new, - } - - def initialize(options={}) - super() - @options = options.merge!(DETECT_LINES_DEFAULTS) - self.clear! - OPERATOR_PROCESSORS.each { |k,v| registerOperatorProcessor(k, v) } - end - - def clear! - self.rulings = [] - self.currentX = -1 - self.currentY = -1 - self.currentPath = [] - @pageSize = nil - end - - def addRuling(ruling, color=nil) - color = color.nil? ? [0,0,0] : color - if !page.getRotation.nil? && [90, -270, -90, 270].include?(page.getRotation) - - mb = page.findMediaBox - - ruling.rotate!(mb.getLowerLeftX, mb.getLowerLeftY, page.getRotation) - - trans = if page.getRotation == 90 || page.getRotation == -270 - AffineTransform.getTranslateInstance(mb.getHeight, 0) - else - AffineTransform.getTranslateInstance(0, mb.getWidth) - end - ruling.transform!(trans) - end - - # snapping to grid and joining lines that are close together - ruling.snap!(options[:snapping_grid_cell_size]) - - self.rulings << [ruling, color] - end - - ## - # get current page size - def pageSize - @pageSize ||= self.page.findMediaBox.createDimension - end - - ## - # fix the Y coordinate based on page rotation - def fixY(y) - pageSize.getHeight - y - end - - def ScaledPoint(*args) - x, y = args[0], args[1] - - # if scale factor not provided, get it from current transformation matrix - if args.size == 2 - ctm = getGraphicsState.getCurrentTransformationMatrix - at = ctm.createAffineTransform - scaleX = at.getScaleX; scaleY = at.getScaleY - else - scaleX = args[2]; scaleY = args[3] - end - - finalX = 0.0; - finalY = 0.0; - - if scaleX > 0 - finalX = x * scaleX; - end - if scaleY > 0 - finalY = y * scaleY; - end - - return java.awt.geom.Point2D::Float.new(finalX, finalY); - - end - - def TransformedPoint(x, y) - position = [x,y].to_java(:float) - at = self.getGraphicsState.getCurrentTransformationMatrix.createAffineTransform - at.transform(position, 0, position, 0, 1) - position[1] = fixY(position[1]) - java.awt.geom.Point2D::Float.new(position[0], position[1]) - end - -end diff --git a/lib/tabula/pdf_render.rb b/lib/tabula/pdf_render.rb deleted file mode 100644 index 7eb3bc4..0000000 --- a/lib/tabula/pdf_render.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'java' - -java_import org.apache.pdfbox.pdmodel.PDDocument -java_import org.apache.pdfbox.pdfviewer.PageDrawer -java_import java.awt.image.BufferedImage -java_import javax.imageio.ImageIO -java_import java.awt.Dimension -java_import java.awt.Color - -module Tabula - module Render - - # render a PDF page to a graphics context, but skip rendering the text - # This is done to reduce 'noise' introduced by the text, we only - # care about lines. - class PageDrawerNoText < PageDrawer - def processTextPosition(text) - end - end - - #ugh jruby; suppresses "ambiguous method" warning that arises due to Java's overloaded constructor. - TRANSPARENT_WHITE = java.awt.Color.java_class.constructor(Java::int, Java::int, Java::int, Java::int).new_instance(255, 255, 255, 0) - - # 2048 width is important, if this is too small, thin lines won't be drawn. - def self.pageToBufferedImage(page, width=2048, pageDrawerClass=PageDrawerNoText) - cropbox = page.findCropBox - widthPt, heightPt = cropbox.getWidth, cropbox.getHeight - pageDimension = Dimension.new(widthPt, heightPt) - rotation = java.lang.Math.toRadians(page.findRotation) - - scaling = width / (rotation == 0 ? widthPt : heightPt) - widthPx, heightPx = (java.lang.Math.java_send :round, [Java::float], widthPt * scaling ), (java.lang.Math.java_send :round, [Java::float], heightPt * scaling) - - - retval = if rotation != 0 - BufferedImage.new(heightPx, widthPx, BufferedImage::TYPE_BYTE_GRAY) - else - BufferedImage.new(widthPx, heightPx, BufferedImage::TYPE_BYTE_GRAY) - end - graphics = retval.getGraphics() - graphics.setBackground(TRANSPARENT_WHITE) - graphics.clearRect(0, 0, retval.getWidth, retval.getHeight) - if rotation != 0 - graphics.java_send :translate, [Java::int, Java::int], retval.getWidth, 0.0 - graphics.rotate(rotation) - end - graphics.scale(scaling, scaling) - drawer = pageDrawerClass.new() - drawer.drawPage(graphics, page, pageDimension) - graphics.dispose - - return retval - end - end -end - -# testing -if __FILE__ == $0 - pdf_file = PDDocument.loadNonSeq(java.io.File.new(ARGV[0]), nil) - bi = Tabula::Render.pageToBufferedImage(pdf_file.getDocumentCatalog.getAllPages[ARGV[1].to_i - 1]) - puts bi.class - ImageIO.write(bi, 'png', - java.io.File.new('notext.png')) -end diff --git a/lib/tabula/spreadsheet_extractor.rb b/lib/tabula/spreadsheet_extractor.rb deleted file mode 100644 index 7e45908..0000000 --- a/lib/tabula/spreadsheet_extractor.rb +++ /dev/null @@ -1,52 +0,0 @@ -module Tabula - module Extraction - - warn 'Tabula::Extraction::SpreadsheetExtractor is DEPRECATED and will be removed. Use ObjectExtractor instead' - - class SpreadsheetExtractor < ObjectExtractor - - # yields each spreadsheet and the page it corresponds to - # because each page can contain an arbitrary number of spreadsheets, each page can be sent - # to the block an arbitrary number of times. - # so the extract.each_with_index trick will absolutely not work. - - # TODO lots of repeated code with parent class - # REFACTOR - def extract(options={}) - Enumerator.new do |y| - begin - @pages.each do |i| - pdfbox_page = @all_pages.get(i-1) #TODO: this can error out ungracefully if you try to extract a page that doesn't exist (e.g. page 5 of a 4 page doc). we should catch and handle. - contents = pdfbox_page.getContents - next if contents.nil? - self.clear! - self.drawPage pdfbox_page - - page = Tabula::Page.new( @pdf_filename, - pdfbox_page.findCropBox.width, - pdfbox_page.findCropBox.height, - pdfbox_page.getRotation.to_i, - i, #one-indexed, just like `i` is. - self.characters, - self.rulings) - - page.spreadsheets(options).each do |spreadsheet| - spreadsheet.cells.each do |cell| - cell.text_elements = page.get_cell_text(cell) - end - y.yield page, spreadsheet - end - end - ensure - @pdf_file.close - end # begin - end - end - end - end -end - - -#new plan: -# find all the cells on the page (lines -> minimal rects) -# find all the spreadsheets from the cells (minimal rects -> maximal rects) diff --git a/lib/tabula/table_extractor.rb b/lib/tabula/table_extractor.rb index 62ed5b1..d899a4a 100644 --- a/lib/tabula/table_extractor.rb +++ b/lib/tabula/table_extractor.rb @@ -1,25 +1,7 @@ +java_import Java::TechnologyTabula::Rectangle module Tabula - - def Tabula.merge_words(text_elements, options={}) - warn 'Tabula.merge_words is DEPRECATED. Use Tabula::TextElement.merge_words instead' - TextElement.merge_words(text_elements, options) - end - - def Tabula.group_by_lines(text_chunks) - warn 'Tabula.group_by_lines is DEPRECATED. Use Tabula::TextChunk.group_by_lines instead.' - TextChunk.group_by_lines(text_chunks) - end - - # Returns an array of Tabula::Line - def Tabula.make_table(page, area, options={}) - warn 'Tabula.make_table is DEPRECATED. Use Tabula::Page#make_table instead.' - page.get_area(area).make_table(options) - end - # extract a table from file +pdf_path+, +pages+ and +area+ # - # +pages+ can be a single integer (1-based) or an array of integers - # # ==== Options # +:password+ - Password if encrypted PDF (default: empty) # +:detect_ruling_lines+ - Try to detect vertical (default: true) @@ -34,18 +16,19 @@ def Tabula.extract_table(pdf_path, page, area, options={}) if area.instance_of?(Array) top, left, bottom, right = area - area = Tabula::ZoneEntity.new(top, left, - right - left, bottom - top) + area = Rectangle.new(top, left, + (right - left), (bottom - top)) end if page.is_a?(Integer) - page = [page] + page = [page.to_java(:int)] end - pdf_page = Extraction::ObjectExtractor.new(pdf_path, - page, - options[:password]) \ - .extract.next + extractor = Extraction::ObjectExtractor.new(pdf_path, + options[:password]) + + pdf_page = extractor.extract(page).next + extractor.close! if ["spreadsheet", "original"].include? options[:extraction_method] use_spreadsheet_extraction_method = options[:extraction_method] == "spreadsheet" @@ -54,39 +37,35 @@ def Tabula.extract_table(pdf_path, page, area, options={}) end if use_spreadsheet_extraction_method - table = pdf_page.get_area(area).spreadsheets.inject(&:+) - else - use_detected_lines = false - if options[:detect_ruling_lines] && options[:vertical_rulings].empty? - detected_vertical_rulings = Ruling.crop_rulings_to_area(pdf_page.vertical_ruling_lines, - area) + spreadsheets = pdf_page.get_area(area).spreadsheets + return spreadsheets.empty? ? Spreadsheet.empty(pdf_page) : spreadsheets.max_by{|s| s.rows.to_a.map(&:size).inject(&:+) } + end - # only use lines if at least 80% of them cover at least 90% - # of the height of area of interest + use_detected_lines = false + if options[:detect_ruling_lines] && options[:vertical_rulings].empty? - # TODO this heuristic SUCKS - # what if only a couple columns is delimited with vertical rulings? - # ie: https://www.dropbox.com/s/lpydler5c3pn408/S2MNCEbirdisland.pdf (see 7th column) - # idea: detect columns without considering rulings, detect vertical rulings - # calculate ratio and try to come up with a threshold - use_detected_lines = detected_vertical_rulings.size > 2 \ - && (detected_vertical_rulings.count { |vl| - vl.height / area.height > 0.9 - } / detected_vertical_rulings.size.to_f) >= 0.8 + detected_vertical_rulings = Ruling.crop_rulings_to_area(pdf_page.vertical_ruling_lines, + area) - end - table = pdf_page.get_area(area).get_table(:vertical_rulings => use_detected_lines ? detected_vertical_rulings : options[:vertical_rulings]) + # only use lines if at least 80% of them cover at least 90% + # of the height of area of interest + + # TODO this heuristic SUCKS + # what if only a couple columns is delimited with vertical rulings? + # ie: https://www.dropbox.com/s/lpydler5c3pn408/S2MNCEbirdisland.pdf (see 7th column) + # idea: detect columns without considering rulings, detect vertical rulings + # calculate ratio and try to come up with a threshold + use_detected_lines = detected_vertical_rulings.size > 2 \ + && (detected_vertical_rulings.count { |vl| + vl.height / area.height > 0.9 + } / detected_vertical_rulings.size.to_f) >= 0.8 - # fixes up the table a little bit, replacing nils with empty TextElements - # and sorting the lines. - table.lines.each do |l| - l.text_elements = l.text_elements.map do |te| - te || TextElement.new(nil, nil, nil, nil, nil, nil, '', nil) - end - end - table.lines.sort_by! { |l| l.text_elements.map { |te| te.top or 0 }.max } - table end + + pdf_page + .get_area(area) + .get_table(:vertical_rulings => use_detected_lines ? detected_vertical_rulings.subList(1, detected_vertical_rulings.size) : options[:vertical_rulings]) + end end diff --git a/lib/tabula/table_guesser.rb b/lib/tabula/table_guesser.rb deleted file mode 100644 index aaec076..0000000 --- a/lib/tabula/table_guesser.rb +++ /dev/null @@ -1,197 +0,0 @@ -require 'json' - -warn 'Tabula::TableGuesser is DEPRECATED and will be removed' - -module Tabula - module TableGuesser - - def TableGuesser.find_and_write_rects(filename, output_dir) - #writes to JSON the rectangles on each page in the specified PDF. - open(File.join(output_dir, "tables.json"), 'w') do |f| - f.write( JSON.dump(find_rects(filename).map{|a| a.map{|r| r.dims.map(&:to_i) }} )) - end - end - - def TableGuesser.find_rects(filename) - pdf = load_pdfbox_pdf(filename) - - if pdf.getNumberOfPages == 0 - puts "not a pdf!" - exit - end - - puts "pages: " + pdf.getNumberOfPages.to_s - - tables = [] - pdf.getNumberOfPages.times do |i| - #gotcha: with PDFView, PDF pages are 1-indexed. If you ask for page 0 and then page 1, you'll get the first page twice. So start with index 1. - tables << find_rects_on_page(pdf, i + 1) - end - tables - end - - def TableGuesser.find_lines(filename) - if pdf.getNumberOfPages == 0 - puts "not a pdf!" - exit - end - - puts "pages: " + pdf.getNumberOfPages.to_s - - lines = [] - pdf.getNumberOfPages.times do |i| - lines << detect_lines_in_pdf_page(filename, i) - end - lines - end - - def TableGuesser.find_lines_on_page(pdf, page_number_zero_indexed) - Tabula::Extraction::LineExtractor.lines_in_pdf_page(pdf, page_number_zero_indexed, {:render_pdf => false}) - end - - def TableGuesser.find_rects_on_page(pdf, page_index) - find_rects_from_lines(find_lines_on_page(pdf, page_index, 10)) - end - - def TableGuesser.find_rects_from_lines(lines) - horizontal_lines = lines.select(&:horizontal?) - vertical_lines = lines.select(&:vertical?) - find_tables(vertical_lines, horizontal_lines).inject([]) do |memo, next_rect| - java.awt.geom.Rectangle2D::Float.unionize( memo, next_rect ) - end.compact.reject{|r| r.area == 0 }.sort_by(&:area).reverse - end - - - def TableGuesser.euclidean_distance(x1, y1, x2, y2) - return Math.sqrt( ((x1 - x2) ** 2) + ((y1 - y2) ** 2) ) - end - - def TableGuesser.is_upward_oriented(line, y_value) - #return true if this line is oriented upwards, i.e. if the majority of it's length is above y_value. - return (y_value - line.top > line.bottom - y_value); - end - - def TableGuesser.find_tables(verticals, horizontals) - # - # Find all the rectangles in the vertical and horizontal lines given. - # - # Rectangles are deduped with hashRectangle, which considers two rectangles identical if each point rounds to the same tens place as the other. - # - # TODO: generalize this. - # - corner_proximity_threshold = 0.005; - - rectangles = [] - #find rectangles with one horizontal line and two vertical lines that end within $threshold to the ends of the horizontal line. - - [true, false].each do |up_or_down_lines| - horizontals.each do |horizontal_line| - horizontal_line_length = horizontal_line.length - - has_vertical_line_from_the_left = false - left_vertical_line = nil - #for the left vertical line. - verticals.each do |vertical_line| - #1. if it is correctly oriented (up or down) given the outer loop here. (We don't want a false-positive rectangle with one "arm" going down, and one going up.) - next unless is_upward_oriented(vertical_line, horizontal_line.top) == up_or_down_lines - - vertical_line_length = vertical_line.length - longer_line_length = [horizontal_line_length, vertical_line_length].max - corner_proximity = corner_proximity_threshold * longer_line_length - #make this the left vertical line: - #2. if it begins near the left vertex of the horizontal line. - if euclidean_distance(horizontal_line.left, horizontal_line.top, vertical_line.left, vertical_line.top) < corner_proximity || - euclidean_distance(horizontal_line.left, horizontal_line.top, vertical_line.left, vertical_line.bottom) < corner_proximity - #3. if it is farther to the left of the line we already have. - if left_vertical_line.nil? || left_vertical_line.left> vertical_line.left #is this line is more to the left than left_vertical_line. #"What's your opinion on Das Kapital?" - has_vertical_line_from_the_left = true - left_vertical_line = vertical_line - end - end - end - - has_vertical_line_from_the_right = false; - right_vertical_line = nil - #for the right vertical line. - verticals.each do |vertical_line| - next unless is_upward_oriented(vertical_line, horizontal_line.top) == up_or_down_lines - vertical_line_length = vertical_line.length - longer_line_length = [horizontal_line_length, vertical_line_length].max - corner_proximity = corner_proximity_threshold * longer_line_length - if euclidean_distance(horizontal_line.right, horizontal_line.top, vertical_line.left, vertical_line.top) < corner_proximity || - euclidean_distance(horizontal_line.right, horizontal_line.top, vertical_line.left, vertical_line.bottom) < corner_proximity - - if right_vertical_line.nil? || right_vertical_line.right > vertical_line.right #is this line is more to the right than right_vertical_line. #"Can you recite all of John Galt's speech?" - #do two passes to guarantee we don't get a horizontal line with a upwards and downwards line coming from each of its corners. - #i.e. ensuring that both "arms" of the rectangle have the same orientation (up or down). - has_vertical_line_from_the_right = true - right_vertical_line = vertical_line - end - end - end - - if has_vertical_line_from_the_right && has_vertical_line_from_the_left - #in case we eventually tolerate not-quite-vertical lines, this computers the distance in Y directly, rather than depending on the vertical lines' lengths. - height = [left_vertical_line.bottom - left_vertical_line.top, right_vertical_line.bottom - right_vertical_line.top].max - - top = [left_vertical_line.top, right_vertical_line.top].min - width = horizontal_line.right - horizontal_line.left - left = horizontal_line.left - r = java.awt.geom.Rectangle2D::Float.new( left, top, width, height ) #x, y, w, h - #rectangles.put(hashRectangle(r), r); #TODO: I dont' think I need this now that I'm in Rubyland - rectangles << r - end - end - - #find rectangles with one vertical line and two horizontal lines that end within $threshold to the ends of the vertical line. - verticals.each do |vertical_line| - vertical_line_length = vertical_line.length - - has_horizontal_line_from_the_top = false - top_horizontal_line = nil - #for the top horizontal line. - horizontals.each do |horizontal_line| - horizontal_line_length = horizontal_line.length - longer_line_length = [horizontal_line_length, vertical_line_length].max - corner_proximity = corner_proximity_threshold * longer_line_length - - if euclidean_distance(vertical_line.left, vertical_line.top, horizontal_line.left, horizontal_line.top) < corner_proximity || - euclidean_distance(vertical_line.left, vertical_line.top, horizontal_line.right, horizontal_line.top) < corner_proximity - if top_horizontal_line.nil? || top_horizontal_line.top > horizontal_line.top #is this line is more to the top than the one we've got already. - has_horizontal_line_from_the_top = true; - top_horizontal_line = horizontal_line; - end - end - end - has_horizontal_line_from_the_bottom = false; - bottom_horizontal_line = nil - #for the bottom horizontal line. - horizontals.each do |horizontal_line| - horizontal_line_length = horizontal_line.length - longer_line_length = [horizontal_line_length, vertical_line_length].max - corner_proximity = corner_proximity_threshold * longer_line_length - - if euclidean_distance(vertical_line.left, vertical_line.bottom, horizontal_line.left, horizontal_line.top) < corner_proximity || - euclidean_distance(vertical_line.left, vertical_line.bottom, horizontal_line.left, horizontal_line.top) < corner_proximity - if bottom_horizontal_line.nil? || bottom_horizontal_line.bottom > horizontal_line.bottom #is this line is more to the bottom than the one we've got already. - has_horizontal_line_from_the_bottom = true; - bottom_horizontal_line = horizontal_line; - end - end - end - - if has_horizontal_line_from_the_bottom && has_horizontal_line_from_the_top - x = [top_horizontal_line.left, bottom_horizontal_line.left].min - y = vertical_line.top - width = [top_horizontal_line.right - top_horizontal_line.left, bottom_horizontal_line.right - bottom_horizontal_line.right].max - height = vertical_line.bottom - vertical_line.top - r = java.awt.geom.Rectangle2D::Float.new( x, y, width, height ) #x, y, w, h - #rectangles.put(hashRectangle(r), r); - rectangles << r - end - end - end - return rectangles.uniq &:similarity_hash - end - end -end diff --git a/lib/tabula/version.rb b/lib/tabula/version.rb index 906f9b2..b85a4ec 100644 --- a/lib/tabula/version.rb +++ b/lib/tabula/version.rb @@ -1,3 +1,3 @@ module Tabula - VERSION = '0.7.2' + VERSION = '1.0.0-alpha' end diff --git a/lib/tabula/writers.rb b/lib/tabula/writers.rb deleted file mode 100644 index 4d4c43b..0000000 --- a/lib/tabula/writers.rb +++ /dev/null @@ -1,29 +0,0 @@ -require 'csv' -require 'json' - -module Tabula - module Writers - - def Writers.CSV(lines, output=$stdout) - lines.each do |l| - output.write CSV.generate_line(l.map(&:text), row_sep: "\r\n") - end - end - - def Writers.JSON(lines, output=$stdout) - output.write lines.to_json - end - - def Writers.TSV(lines, output=$stdout) - lines.each do |l| - output.write CSV.generate_line(l.map(&:text), col_sep: "\t", row_sep: "\r\n") - end - end - - def Writers.HTML(lines, output=$stdout) - raise "not implemented" - end - - - end -end diff --git a/tabula-extractor.gemspec b/tabula-extractor.gemspec index ec06c81..e756e2e 100644 --- a/tabula-extractor.gemspec +++ b/tabula-extractor.gemspec @@ -2,7 +2,6 @@ $:.push File.expand_path("../lib", __FILE__) require 'tabula/version' - Gem::Specification.new do |s| s.name = "tabula-extractor" s.version = Tabula::VERSION @@ -15,17 +14,13 @@ Gem::Specification.new do |s| s.platform = 'java' - shared_libs = ['liblsd.dylib', 'liblsd-linux64.so', 'liblsd-linux32.so', 'liblsd.dll', 'liblsd64.dll'].map { |f| 'ext/' + f } - s.files = `git ls-files`.split("\n") + shared_libs.map.reject { |f| !File.exists?(f) } - s.test_files = `git ls-files -- {test,features}/*`.split("\n") + s.files = `git ls-files`.split("\n").reject { |f| f =~ /^test\// } s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } s.require_paths = ["lib"] - s.add_development_dependency 'minitest' - s.add_development_dependency 'bundler', '>= 1.3.4' - s.add_development_dependency 'ruby-debug' - s.add_development_dependency 'pry' + s.add_development_dependency 'ruby-debug', '~> 0.10.4' + s.add_development_dependency 'pry', '~> 0.10.1' + s.add_development_dependency 'minitest', '~> 5.4.0' s.add_runtime_dependency "trollop", ["~> 2.0"] -# s.add_runtime_dependency "algorithms", ["~> 0.6.1"] end diff --git a/target/jsi-1.1.0-SNAPSHOT.jar b/target/jsi-1.1.0-SNAPSHOT.jar deleted file mode 100644 index 60bfa61..0000000 Binary files a/target/jsi-1.1.0-SNAPSHOT.jar and /dev/null differ diff --git a/target/slf4j-api-1.6.3.jar b/target/slf4j-api-1.6.3.jar deleted file mode 100644 index 08a6c80..0000000 Binary files a/target/slf4j-api-1.6.3.jar and /dev/null differ diff --git a/target/pdfbox-app-2.0.0-SNAPSHOT.jar b/target/tabula-0.8.0-jar-with-dependencies.jar similarity index 55% rename from target/pdfbox-app-2.0.0-SNAPSHOT.jar rename to target/tabula-0.8.0-jar-with-dependencies.jar index 99ff2ac..6031706 100644 Binary files a/target/pdfbox-app-2.0.0-SNAPSHOT.jar and b/target/tabula-0.8.0-jar-with-dependencies.jar differ diff --git a/target/trove4j-3.0.3.jar b/target/trove4j-3.0.3.jar deleted file mode 100644 index cd00f93..0000000 Binary files a/target/trove4j-3.0.3.jar and /dev/null differ diff --git a/test/data/brazil_crop_area.pdf b/test/data/brazil_crop_area.pdf new file mode 100644 index 0000000..05dda46 Binary files /dev/null and b/test/data/brazil_crop_area.pdf differ diff --git a/test/data/french1.pdf b/test/data/french1.pdf new file mode 100644 index 0000000..54501d7 Binary files /dev/null and b/test/data/french1.pdf differ diff --git a/test/data/french1.tsv b/test/data/french1.tsv new file mode 100644 index 0000000..840a803 --- /dev/null +++ b/test/data/french1.tsv @@ -0,0 +1,43 @@ +AFRIQUE DU SUD • PAP SINCLAIR BEILES "" "" "" "" +COLLECTIF Picador Book of African Stories Divers L Stephen Gray +QUIGNARD PASCAL Tous les matins du monde Gallimard L Protea Boekhuis +ALBANIE • PAP FAN NOLI "" "" "" "" +COLLECTIF Dictionnaire de la philosophie Larousse P Shtepia Botuese Enciklopedike +ADLER LAURE Marguerite Duras Gallimard L Albin +ALLÈGRE CLAUDE Introduction à une histoire naturelle Fayard SSH Dituria +AYMÉ MARCEL La Jument verte Gallimard L Albin +"" Le Passe-Muraille Gallimard L Dituria +BATAILLE GEORGES L’Érotisme Minuit L Shtepia E Librit Dhe E Komunikimit +BEAUVOIR SIMONE (DE) L’Âge de discrétion Gallimard L Elena Gjika +"" La Femme rompue Gallimard L Elena Gjika +"" Monologue Gallimard L Elena Gjika +BECKETT SAMUEL En attendant Godot Minuit L MCM +"" Oh les beaux jours Minuit L MCM +BEN JELLOUN TAHAR L’Enfant de sable Le Seuil L Elena Gjika +BERGSON HENRI Les Deux Sources de la morale et de la religion PUF P Amfioni dhe Zeti +BERNHEIM EMMANUÈLE Sa femme Gallimard L Dituria +BOSQUET ALAIN Demain sans moi Gallimard L Toena +"" Le Livre du doute et de la grâce Gallimard L Toena +CABANES PIERRE Passions albanaises, de Berisha au Kosovo Odile Jacob SSH Onufri +CAMUS ALBERT La Chute Gallimard L Naim Frasheri +"" Le Mythe de Sisyphe Gallimard L Fan Noli +CHAILLEY JACQUES 40000 Ans de musique L’Harmattan A ISCM +CHAMPSEIX ÉLISABETH & CHAMPSEIX JEAN-PAUL L’Albanie ou la Logique du désespoir La Découverte SSH Elena Gjika +CORNEILLE PIERRE Horace "" L Toena +CUIN CHARLES-HENRY & GRESLE FRANÇOIS Histoire de la sociologie La Découverte SSH Shpiragu +DEBRAY RÉGIS Dieu, un itinéraire Odile Jacob SSH Ora Editions +DELERM PHILIPPE La Première Gorgée de bière et autres plaisirs miniscules Gallimard L Ombra GVG +DOLTO FRANÇOISE Lorsque l’enfant paraît Le Seuil SSH Kthim në Vetvete +"" Paroles pour adolescents Hatier SSH Fan Noli +DURAS MARGUERITE L’Amant Minuit L Dituria +"" Dix Heures et demie du soir en été Gallimard L Dituria +GENET JEAN Journal du voleur Gallimard L Apollonia +GIL ROGER Neuropsychologie Masson ST Dituria +GIONO JEAN Regain Grasset L Albin +"" Un de Baumugnes Grasset L Globus +GRACQ JULIEN Au château d’Argol José Corti L Globus +KOKONA VEDAT Anthologie bilingue de poésie française "" L Toena +"" Dictionnaire albanais-français "" L Toena +KUNDERA MILAN La Plaisanterie Gallimard L Dituria +LE CLÉZIO J.-M. G. Étoile errante Gallimard L Albin +"" La Ronde et autres récits (extraits) Gallimard L Dituria diff --git a/test/data/frx_2012_disclosure.tsv b/test/data/frx_2012_disclosure.tsv index d9bff82..2b33904 100644 --- a/test/data/frx_2012_disclosure.tsv +++ b/test/data/frx_2012_disclosure.tsv @@ -1,88 +1,88 @@ -FOREST LABORATORIES, INC. DISCLOSURE REPORT "" "" "" "" -Calendar Year - 2012 "" "" "" "" +FOREST LABORATORIES, INC. DISCLOSURE REPORT +Calendar Year - 2012 Physician Related Entity (if applicable) City / State Purpose of Payment Amount ($USD) * ** -AALAEI, BEHZAD "" HIGHLAND, IN MEALS $51.24 -TOTAL "" "" "" $51.24 -AAMODT, DENISE, E "" ALBUQUERQUE, NM MEALS $66.12 -TOTAL "" "" "" $66.12 -AANONSEN, DEBORAH, A "" STATEN ISLAND, NY MEALS $85.00 -TOTAL "" "" "" $85.00 -AARON, CAREN, T "" RICHMOND, VA EDUCATIONAL ITEMS $78.80 -AARON, CAREN, T "" RICHMOND, VA MEALS $392.45 -TOTAL "" "" "" $471.25 -AARON, JOHN "" CLARKSVILLE, TN MEALS $20.39 -TOTAL "" "" "" $20.39 -AARON, JOSHUA, N "" WEST GROVE, PA MEALS $310.33 -AARON, JOSHUA, N "REGIONAL PULMONARY & SLEEP MEDICINE" WEST GROVE, PA SPEAKING FEES $4,700.00 -TOTAL "" "" "" $5,010.33 -AARON, MAUREEN, M "" MARTINSVILLE, VA MEALS $193.67 -TOTAL "" "" "" $193.67 -AARON, MICHAEL, L "" WEST ISLIP, NY MEALS $19.50 -TOTAL "" "" "" $19.50 -AARON, MICHAEL, R "" BROOKLYN, NY MEALS $65.92 -TOTAL "" "" "" $65.92 -AARONS, MARK, G "" PINEHURST, NC MEALS $154.19 -TOTAL "" "" "" $154.19 -AARONSON, GARY, A "" PHILADELPHIA, PA MEALS $205.17 -TOTAL "" "" "" $205.17 -AARONSON, ROBERT, M "" TUCSON, AZ MEALS $24.38 -TOTAL "" "" "" $24.38 -AASHEIM, RICHARD, J "" GREENEVILLE, TN EDUCATIONAL ITEMS $2.27 -AASHEIM, RICHARD, J "" GREENEVILLE, TN MEALS $100.76 -TOTAL "" "" "" $103.03 -AASMAA, SIRIKE, T "" MONTVILLE, NJ MEALS $53.33 -TOTAL "" "" "" $53.33 -AAZAMI, HESSAM "" GRANADA HILLS, CA MEALS $402.90 -TOTAL "" "" "" $402.90 -ABAABA, ABIEDU, C "" JACKSONVILLE, FL MEALS $13.49 -TOTAL "" "" "" $13.49 -ABABNEH, ALAELDIN, A "" KANSAS CITY, KS MEALS $10.31 -TOTAL "" "" "" $10.31 -ABAD, ANTONIO, A "" CORAL SPRINGS, FL MEALS $516.29 -TOTAL "" "" "" $516.29 -ABADEER, PETER, S "" NORMAL, IL MEALS $200.38 -TOTAL "" "" "" $200.38 -ABAD, ENZO, L "" MIAMI, FL MEALS $67.61 -TOTAL "" "" "" $67.61 -"ABADIAN SHARIFABAD, MANOOCHEHR" "" GRANADA HILLS, CA MEALS $12.37 -TOTAL "" "" "" $12.37 -ABADI, CHRISTOPHER, A "" WARWICK, RI MEALS $157.42 -TOTAL "" "" "" $157.42 -ABADIE, MARCUS, G "" ATHENS, TX MEALS $361.89 -TOTAL "" "" "" $361.89 -ABADI, JAMSHEED, S "" BROOKLYN, NY MEALS $363.40 -TOTAL "" "" "" $363.40 -ABADILLA, JUNE, E "" JACKSON, KY MEALS $105.33 -TOTAL "" "" "" $105.33 -ABAD, JOHN, P "" NEWARK, OH MEALS $347.64 -TOTAL "" "" "" $347.64 -ABAD, JOSE, F "" FOLSOM, CA MEALS $30.28 -TOTAL "" "" "" $30.28 -ABAD, REMEDIOS, D "" WILNINGTON, DE MEALS $26.85 -TOTAL "" "" "" $26.85 -ABAD, SO KIM, F "" WICHITA FALLS, TX MEALS $136.52 -TOTAL "" "" "" $136.52 -ABAD, ZOILO, R "" MIAMI, FL MEALS $93.83 -TOTAL "" "" "" $93.83 -ABALIHI, CAROL, N "" EL PASO, TX MEALS $88.48 -TOTAL "" "" "" $88.48 -ABALOS, ANNA, T "" ROSEVILLE, CA MEALS $178.60 -TOTAL "" "" "" $178.60 -ABALOS, ARTURO, Z "" DELANO, CA MEALS $48.06 -TOTAL "" "" "" $48.06 -ABALOS, JOSEPH, M "" SENECA, PA MEALS $39.03 -TOTAL "" "" "" $39.03 -ABANDO, JOSE, R "" DAYTONA BEACH, FL MEALS $83.44 -TOTAL "" "" "" $83.44 -ABANG, ANTHONY, E "" ELIZABETHTOWN, KY MEALS $12.62 -TOTAL "" "" "" $12.62 -ABAN, KENRIC, T "" SAN DIEGO, CA MEALS $11.91 -TOTAL "" "" "" $11.91 -ABAQUETA, ALVIN, Y "" CHARLOTTE, NC MEALS $233.71 -TOTAL "" "" "" $233.71 -ABARCA, SERGIO, O "" TOOELE, UT MEALS $159.58 -TOTAL "" "" "" $159.58 -ABARIKWU, CONSTANTIA, A "" PHOENIX, AZ MEALS $153.57 -TOTAL "" "" "" $153.57 -ABASHIDZE, TEAH, A "" CLEVELAND, OH MEALS $153.59 -TOTAL "" "" "" $153.59 +AALAEI, BEHZAD HIGHLAND, IN MEALS $51.24 +TOTAL $51.24 +AAMODT, DENISE, E ALBUQUERQUE, NM MEALS $66.12 +TOTAL $66.12 +AANONSEN, DEBORAH, A STATEN ISLAND, NY MEALS $85.00 +TOTAL $85.00 +AARON, CAREN, T RICHMOND, VA EDUCATIONAL ITEMS $78.80 +AARON, CAREN, T RICHMOND, VA MEALS $392.45 +TOTAL $471.25 +AARON, JOHN CLARKSVILLE, TN MEALS $20.39 +TOTAL $20.39 +AARON, JOSHUA, N WEST GROVE, PA MEALS $310.33 +AARON, JOSHUA, N "REGIONAL PULMONARY & SLEEP MEDICINE" WEST GROVE, PA SPEAKING FEES $4,700.00 +TOTAL $5,010.33 +AARON, MAUREEN, M MARTINSVILLE, VA MEALS $193.67 +TOTAL $193.67 +AARON, MICHAEL, L WEST ISLIP, NY MEALS $19.50 +TOTAL $19.50 +AARON, MICHAEL, R BROOKLYN, NY MEALS $65.92 +TOTAL $65.92 +AARONS, MARK, G PINEHURST, NC MEALS $154.19 +TOTAL $154.19 +AARONSON, GARY, A PHILADELPHIA, PA MEALS $205.17 +TOTAL $205.17 +AARONSON, ROBERT, M TUCSON, AZ MEALS $24.38 +TOTAL $24.38 +AASHEIM, RICHARD, J GREENEVILLE, TN EDUCATIONAL ITEMS $2.27 +AASHEIM, RICHARD, J GREENEVILLE, TN MEALS $100.76 +TOTAL $103.03 +AASMAA, SIRIKE, T MONTVILLE, NJ MEALS $53.33 +TOTAL $53.33 +AAZAMI, HESSAM GRANADA HILLS, CA MEALS $402.90 +TOTAL $402.90 +ABAABA, ABIEDU, C JACKSONVILLE, FL MEALS $13.49 +TOTAL $13.49 +ABABNEH, ALAELDIN, A KANSAS CITY, KS MEALS $10.31 +TOTAL $10.31 +ABAD, ANTONIO, A CORAL SPRINGS, FL MEALS $516.29 +TOTAL $516.29 +ABADEER, PETER, S NORMAL, IL MEALS $200.38 +TOTAL $200.38 +ABAD, ENZO, L MIAMI, FL MEALS $67.61 +TOTAL $67.61 +"ABADIAN SHARIFABAD, MANOOCHEHR" GRANADA HILLS, CA MEALS $12.37 +TOTAL $12.37 +ABADI, CHRISTOPHER, A WARWICK, RI MEALS $157.42 +TOTAL $157.42 +ABADIE, MARCUS, G ATHENS, TX MEALS $361.89 +TOTAL $361.89 +ABADI, JAMSHEED, S BROOKLYN, NY MEALS $363.40 +TOTAL $363.40 +ABADILLA, JUNE, E JACKSON, KY MEALS $105.33 +TOTAL $105.33 +ABAD, JOHN, P NEWARK, OH MEALS $347.64 +TOTAL $347.64 +ABAD, JOSE, F FOLSOM, CA MEALS $30.28 +TOTAL $30.28 +ABAD, REMEDIOS, D WILNINGTON, DE MEALS $26.85 +TOTAL $26.85 +ABAD, SO KIM, F WICHITA FALLS, TX MEALS $136.52 +TOTAL $136.52 +ABAD, ZOILO, R MIAMI, FL MEALS $93.83 +TOTAL $93.83 +ABALIHI, CAROL, N EL PASO, TX MEALS $88.48 +TOTAL $88.48 +ABALOS, ANNA, T ROSEVILLE, CA MEALS $178.60 +TOTAL $178.60 +ABALOS, ARTURO, Z DELANO, CA MEALS $48.06 +TOTAL $48.06 +ABALOS, JOSEPH, M SENECA, PA MEALS $39.03 +TOTAL $39.03 +ABANDO, JOSE, R DAYTONA BEACH, FL MEALS $83.44 +TOTAL $83.44 +ABANG, ANTHONY, E ELIZABETHTOWN, KY MEALS $12.62 +TOTAL $12.62 +ABAN, KENRIC, T SAN DIEGO, CA MEALS $11.91 +TOTAL $11.91 +ABAQUETA, ALVIN, Y CHARLOTTE, NC MEALS $233.71 +TOTAL $233.71 +ABARCA, SERGIO, O TOOELE, UT MEALS $159.58 +TOTAL $159.58 +ABARIKWU, CONSTANTIA, A PHOENIX, AZ MEALS $153.57 +TOTAL $153.57 +ABASHIDZE, TEAH, A CLEVELAND, OH MEALS $153.59 +TOTAL $153.59 diff --git a/test/data/gretna-owh-request.pdf b/test/data/gretna-owh-request.pdf new file mode 100644 index 0000000..1a36269 Binary files /dev/null and b/test/data/gretna-owh-request.pdf differ diff --git a/test/data/mineria.pdf b/test/data/mineria.pdf new file mode 100644 index 0000000..2b4304e Binary files /dev/null and b/test/data/mineria.pdf differ diff --git a/test/data/monospaced_ascii_sep.pdf b/test/data/monospaced_ascii_sep.pdf new file mode 100644 index 0000000..3499381 Binary files /dev/null and b/test/data/monospaced_ascii_sep.pdf differ diff --git a/test/data/spanning_cells.csv b/test/data/spanning_cells.csv index 948d721..9d7a5a9 100644 --- a/test/data/spanning_cells.csv +++ b/test/data/spanning_cells.csv @@ -1,18 +1,18 @@ -Improved operation scenario,"","","","","" +Improved operation scenario,,,,, Volume servers in:,2007,2008,2009,2010,2011 Server closets,"1,505","1,580","1,643","1,673","1,689" Server rooms,"1,512","1,586","1,646","1,677","1,693" Localized data centers,"1,512","1,586","1,646","1,677","1,693" Mid-tier data centers,"1,512","1,586","1,646","1,677","1,693" Enterprise-class data centers,"1,512","1,586","1,646","1,677","1,693" -Best practice scenario,"","","","","" +Best practice scenario,,,,, Volume servers in:,2007,2008,2009,2010,2011 Server closets,"1,456","1,439","1,386","1,296","1,326" Server rooms,"1,465","1,472","1,427","1,334","1,371" Localized data centers,"1,465","1,471","1,426","1,334","1,371" Mid-tier data centers,"1,465","1,471","1,426","1,334","1,371" Enterprise-class data centers,"1,465","1,471","1,426","1,334","1,371" -State-of-the-art scenario,"","","","","" +State-of-the-art scenario,,,,, Volume servers in:,2007,2008,2009,2010,2011 Server closets,"1,485","1,471","1,424","1,315","1,349" Server rooms,"1,495","1,573","1,586","1,424","1,485" diff --git a/test/heuristic-test-set/original/cap1cu04.pdf b/test/heuristic-test-set/original/cap1cu04.pdf new file mode 100644 index 0000000..b849144 Binary files /dev/null and b/test/heuristic-test-set/original/cap1cu04.pdf differ diff --git a/test/heuristic.rb b/test/heuristic.rb index b207b7d..11a8099 100644 --- a/test/heuristic.rb +++ b/test/heuristic.rb @@ -30,6 +30,7 @@ def heuristic(page) elsif page_is_tabular && !expected_to_be_tabular misclassified_as_spreadsheet << filename elsif !page_is_tabular && expected_to_be_tabular + puts page.heuristic_ratio misclassified_as_original << filename end end diff --git a/test/icdar-test.rb b/test/icdar-test.rb new file mode 100644 index 0000000..66ef861 --- /dev/null +++ b/test/icdar-test.rb @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +require 'nokogiri' +require_relative '../lib/tabula' + +module ICDARTest + class Table < Struct.new(:id, :filename) + attr_accessor :regions + end + + class Region < Struct.new(:id, + :page, + :col_increment, :row_increment, + :x1, :x2, :y1, :y2) + attr_accessor :cells + + end + + + class Cell < Struct.new(:id, + :start_col, :end_col, + :start_row, :end_row, + :x1, :x2, :y1, :y2, + :content) + + def end_col + self[:end_col] || self[:start_col] + end + + def end_row + self[:end_row] || self[:start_row] + end + + end +end + +def parse_structure_groundtruth(path) + xml = Nokogiri::XML(File.open(path)) + filename = File.expand_path(xml.xpath('/document/@filename').to_s, File.dirname(path)) + xml.xpath('/document/table').map do |table_el| + table = ICDARTest::Table.new(table_el.attr('id'), filename) + table.regions = table_el.xpath('region').map do |region_el| + + region = ICDARTest::Region.new(region_el.attr('id'), + region_el.attr('page').to_i, + region_el.attr('col-increment').to_i, + region_el.attr('row-increment').to_i) + + region.cells = region_el.xpath('cell').map do |cell_el| + bbox_el = cell_el.xpath('bounding-box').first + content_el = cell_el.xpath('content') + + ICDARTest::Cell.new(cell_el.attr('id'), + cell_el.attr('start-col').to_i, cell_el.attr('end-col').to_i, + cell_el.attr('start-row').to_i, cell_el.attr('end-row').to_i, + bbox_el.attr('x1').to_f, bbox_el.attr('x2').to_f, bbox_el.attr('y1').to_f, bbox_el.attr('y2').to_f, + content_el.text) + end + region + end + table + end +end + +def parse_region_groundtruth(path) + xml = Nokogiri::XML(File.open(path)) + filename = File.expand_path(xml.xpath('/document/@filename').to_s, File.dirname(path)) + xml.xpath('/document/table').map do |table_el| + table = ICDARTest::Table.new(table_el.attr('id'), filename) + table.regions = table_el.xpath('region').map do |region_el| + bbox_el = region_el.xpath('bounding-box').first + ICDARTest::Region.new(region_el.attr('id'), + region_el.attr('page').to_i, + nil, nil, + bbox_el.attr('x1').to_f, + bbox_el.attr('x2').to_f, + bbox_el.attr('y1').to_f, + bbox_el.attr('y2').to_f) + end + table + end +end + +def run_test(id) + dir = id.start_with?('eu') ? 'eu-dataset' : 'us-gov-dataset' + + structure = parse_structure_groundtruth(File.expand_path(File.join('data/icdar-groundtruth', dir, id + '-str.xml'), File.dirname(__FILE__))) + region = parse_region_groundtruth(File.expand_path(File.join('data/icdar-groundtruth', dir, id + '-reg.xml'), File.dirname(__FILE__))) + + structure.zip(region).each do |str, reg| + str.regions.zip(reg.regions).each do |str_reg, reg_reg| + # need to invert y-coords + extractor = Tabula::Extraction::ObjectExtractor.new(reg.filename) + page = extractor.extract(reg_reg.page) + extractor.close! + area = [page.getHeight - reg_reg.y2, + reg_reg.x1, + page.getHeight - reg_reg.y1, + reg_reg.x2] + #puts "java -Djava.awt.headless=true -cp /Users/manuel/Work/tabula/tabula-extractor/ext/tabula/target/tabula-extractor-0.7.4-SNAPSHOT-jar-with-dependencies.jar org.nerdpower.tabula.debug.Debug -a #{area.join(',')} -g -f -p #{reg_reg.page} #{reg.filename}" + puts + # puts Tabula.extract_table(reg.filename, + # reg_reg.page, + # area).to_csv + # puts '----------------------------------------' + end + end +end + +run_test(ARGV.first) diff --git a/test/test_bin_tabula.sh b/test/test_bin_tabula.sh old mode 100644 new mode 100755 index f18e5c8..0e31a32 --- a/test/test_bin_tabula.sh +++ b/test/test_bin_tabula.sh @@ -1,7 +1,7 @@ -bin/tabula test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf --silent -o test.csv -bin/tabula test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf -o test.csv -bin/tabula test/heuristic-test-set/original/bo_page24.pdf -o test.csv -bin/tabula test/heuristic-test-set/original/bo_page24.pdf -o test.csv --format TSV -bin/tabula test/data/campaign_donors.pdf -o test.csv --columns 47,147,256,310,375,431,504 #columns should work -bin/tabula test/data/argentina_diputados_voting_record.pdf --guess -o test.csv --format TSV #should exclude guff -bin/tabula test/data/vertical_rulings_bug.pdf --area 250,0,325,1700 -o test.csv --format TSV #should be only a few lines \ No newline at end of file +ruby bin/tabula test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf --silent -o test.csv +ruby bin/tabula test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf -o test.csv +ruby bin/tabula test/heuristic-test-set/original/bo_page24.pdf -o test.csv +ruby bin/tabula test/heuristic-test-set/original/bo_page24.pdf -o test.csv --format TSV +ruby bin/tabula test/data/campaign_donors.pdf -o test.csv --columns 47,147,256,310,375,431,504 #columns should work +ruby bin/tabula test/data/argentina_diputados_voting_record.pdf --guess -o test.csv --format TSV #should exclude guff +ruby bin/tabula test/data/vertical_rulings_bug.pdf --area 250,0,325,1700 -o test.csv --format TSV #should be only a few lines diff --git a/test/tests.rb b/test/tests.rb old mode 100644 new mode 100755 index 12c2295..b4c4170 --- a/test/tests.rb +++ b/test/tests.rb @@ -1,8 +1,11 @@ +#!/usr/bin/env jruby -J-Djava.awt.headless=true # -*- coding: utf-8 -*- require 'minitest' require 'minitest/autorun' +require 'csv' require_relative '../lib/tabula' +java_import Java::TechnologyTabula::Rectangle def table_to_array(table) lines_to_array(table.rows) @@ -23,7 +26,8 @@ def lines_to_table(lines) module Tabula class Table def inspect - "[" + lines.map(&:inspect).join(",") + "]" + getRows.map { |row| row.map(&:getText).join(",") } + #"[" + lines.map(&:inspect).join(",") + "]" end end end @@ -39,15 +43,15 @@ def inspect class TestEntityComparability < Minitest::Test def test_text_element_comparability - base = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, "Jeremy", nil) + base = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, "Jeremy", 0, 0) - two = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, " Jeremy \n", nil) - three = Tabula::TextElement.new(7, 6, 8, 6, nil, 12, "Jeremy", 88) - four = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "Jeremy", 55) + two = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, " Jeremy \n", 0, 0) + three = Tabula::TextElement.new(7, 6, 8, 6, nil, 12, "Jeremy", 88, 0) + four = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "Jeremy", 55, 0) - five = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "jeremy b", 55) - six = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "jeremy kj", 55) - seven = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, "jeremy kj", nil) + five = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "jeremy b", 55, 0) + six = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "jeremy kj", 55, 0) + seven = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, "jeremy kj", 55, 0) assert_equal base, two assert_equal base, three assert_equal base, four @@ -58,15 +62,15 @@ def test_text_element_comparability end def test_line_comparability - text_base = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, "Jeremy", nil) + text_base = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, "Jeremy", 0) - text_two = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, " Jeremy \n", nil) + text_two = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, " Jeremy \n", 0) text_three = Tabula::TextElement.new(7, 6, 8, 6, nil, 12, "Jeremy", 88) - text_four = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "Jeremy", 55) + text_four = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "Jeremy", 55) - text_five = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "jeremy b", 55) - text_six = Tabula::TextElement.new(5, 7, 1212, 121, 66, 15, "jeremy kj", 55) - text_seven = Tabula::TextElement.new(nil, nil, nil, nil, nil, nil, "jeremy kj", nil) + text_five = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "jeremy b", 55) + text_six = Tabula::TextElement.new(5, 7, 1212, 121, nil, 15, "jeremy kj", 55) + text_seven = Tabula::TextElement.new(0, 0, 0, 0, nil, 0, "jeremy kj", 0) line_base = Tabula::Line.new line_base.text_elements = [text_base, text_two, text_three] line_equal = Tabula::Line.new @@ -86,31 +90,6 @@ def test_line_comparability refute_equal line_base, line_unequal_and_longer refute_equal line_base, line_unequal_and_longer_and_different end - - def test_table_comparability - rows_base = [["a", "b", "c"], ['', 'd', '']] - rows_equal = [["a", "b", "c"], ['', 'd']] - rows_equal_padded = [['', "a", "b", "c"], ['', '', 'd']] - rows_unequal_one = [["a", "b", "c"], ['d']] - rows_unequal_two = [["a", "b", "c"], ['d', '']] - rows_unequal_three = [["a", "b", "c"], ['d'], ['a','b', 'd']] - rows_unequal_four = [["a", "b", "c"]] - - table_base = Tabula::Table.new_from_array(rows_base) - table_equal = Tabula::Table.new_from_array(rows_equal) - table_equal_column_padded = Tabula::Table.new_from_array(rows_equal_padded) - table_unequal_one = Tabula::Table.new_from_array(rows_unequal_one) - table_unequal_two = Tabula::Table.new_from_array(rows_unequal_two) - table_unequal_three = Tabula::Table.new_from_array(rows_unequal_three) - table_unequal_four = Tabula::Table.new_from_array(rows_unequal_four) - - assert_equal table_base, table_equal - assert_equal table_base, table_equal_column_padded - refute_equal table_base, table_unequal_one - refute_equal table_base, table_unequal_two - refute_equal table_base, table_unequal_three - refute_equal table_base, table_unequal_four - end end class TestPagesInfoExtractor < Minitest::Test @@ -126,33 +105,21 @@ def test_pages_info_extractor end end -class TestTableGuesser < Minitest::Test - def test_find_rects_from_lines_with_lsd - skip "Skipping until we actually use LSD" - filename = File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) - page_index = 0 - lines = Tabula::Extraction::LineExtractor.lines_in_pdf_page(filename, page_index, :render_pdf => true) - - page_areas = Tabula::TableGuesser::find_rects_from_lines(lines) - page_areas.map!{|rect| rect.dims(:top, :left, :bottom, :right)} - expected_page_areas = [[54.087890625, 50.203125, 734.220703125, 550.44140625]] - assert_equal expected_page_areas, page_areas - end - -end - class TestDumper < Minitest::Test def test_extractor extractor = Tabula::Extraction::ObjectExtractor.new(File.expand_path('data/gre.pdf', File.dirname(__FILE__))) - page = extractor.extract.next + page = extractor.extract.first + extractor.close! assert_instance_of Tabula::Page, page end def test_get_by_area extractor = Tabula::Extraction::ObjectExtractor.new(File.expand_path('data/gre.pdf', File.dirname(__FILE__))) - characters = extractor.extract.next.get_text([107.1, 57.9214, 394.5214, 290.7]) - assert_equal characters.size, 206 + page = extractor.extract.first + characters = page.get_text(107.1, 60.10, 313.65, 291.79) + extractor.close! + assert_equal 151, characters.size end end @@ -176,11 +143,50 @@ def test_ruling_intersection class TestExtractor < Minitest::Test + def test_extraction_of_multiple_pages + expected_by_page = [ + [ + ["Last Name", "First Name", "Address", "City", "State", "Zip", "Occupation", "Employer", "Date", "Amount"], + ["Lidstad", "Dick & Peg", "62 Mississippi River Blvd N", "Saint Paul", "MN", "55104", "retired", "", "10/12/2012", "60.00"], + ], + [ + ["Last Name", "First Name", "Address", "City", "State", "Zip", "Occupation", "Employer", "Date", "Amount"], + ["Filice","Gregory A","15 Crocus Place","Saint Paul","MN","55102","Physician","Veterans Affairs Medical Cent","10/3/2012","100.00"] + ], + [ + ["Last Name", "First Name", "Address", "City", "State", "Zip", "Occupation", "Employer", "Date", "Amount"], + ["Skovolt","Glen and Anna","1473 Grantham St.","Saint Paul","MN","55108","retired","","9/12/2012","100.00"] + ] + ] + (1..3).to_a.each do |page_num| + + table = table_to_array Tabula.extract_table(File.expand_path('data/strongschools.pdf', File.dirname(__FILE__)), + page_num, + [52.32857142857143,15.557142857142859,128.70000000000002,767.9571428571429], + :detect_ruling_lines => true) + assert_equal expected_by_page[page_num-1], table[0...2] + end + end + + def test_extraction_of_all_pages + extractor = Tabula::Extraction::ObjectExtractor.new(File.expand_path('data/strongschools.pdf', File.dirname(__FILE__)), :all) + pages = extractor.extract.to_a + + expected_unique_words_per_page = ["Lidstad", "Filice", "Spencer de Gutierrez", "Mancinis", "D'Aquila"] + + pages.each_with_index do |pdf_page, idx| + text_chunks = Tabula::TextElement.merge_words(pdf_page.texts) + page_text_joined = text_chunks.map(&:text).join(" ") + assert page_text_joined.include?(expected_unique_words_per_page[idx]) + end + end + def test_table_extraction_1 table = table_to_array Tabula.extract_table(File.expand_path('data/gre.pdf', File.dirname(__FILE__)), 1, [107.1, 57.9214, 394.5214, 290.7], - :detect_ruling_lines => false) + :detect_ruling_lines => false, + :extraction_method => 'original') expected = [["Prior Scale","New Scale","% Rank*"], ["800","170","99"], ["790","170","99"], ["780","170","99"], ["770","170","99"], ["760","170","99"], ["750","169","99"], ["740","169","99"], ["730","168","98"], ["720","168","98"], ["710","167","97"], ["700","166","96"], ["690","165","95"], ["680","165","95"], ["670","164","93"], ["660","164","93"], ["650","163","91"]] @@ -190,38 +196,22 @@ def test_table_extraction_1 def test_diputados_voting_record table = table_to_array Tabula.extract_table(File.expand_path('data/argentina_diputados_voting_record.pdf', File.dirname(__FILE__)), 1, - [269.875, 12.75, 790.5, 561]) + [269.875, 12.75, 792.5, 565], + :detect_ruling_lines => false) expected = [["ABDALA de MATARAZZO, Norma Amanda", "Frente Cívico por Santiago", "Santiago del Estero", "AFIRMATIVO"], ["ALBRIEU, Oscar Edmundo Nicolas", "Frente para la Victoria - PJ", "Rio Negro", "AFIRMATIVO"], ["ALONSO, María Luz", "Frente para la Victoria - PJ", "La Pampa", "AFIRMATIVO"], ["ARENA, Celia Isabel", "Frente para la Victoria - PJ", "Santa Fe", "AFIRMATIVO"], ["ARREGUI, Andrés Roberto", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["AVOSCAN, Herman Horacio", "Frente para la Victoria - PJ", "Rio Negro", "AFIRMATIVO"], ["BALCEDO, María Ester", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["BARRANDEGUY, Raúl Enrique", "Frente para la Victoria - PJ", "Entre Ríos", "AFIRMATIVO"], ["BASTERRA, Luis Eugenio", "Frente para la Victoria - PJ", "Formosa", "AFIRMATIVO"], ["BEDANO, Nora Esther", "Frente para la Victoria - PJ", "Córdoba", "AFIRMATIVO"], ["BERNAL, María Eugenia", "Frente para la Victoria - PJ", "Jujuy", "AFIRMATIVO"], ["BERTONE, Rosana Andrea", "Frente para la Victoria - PJ", "Tierra del Fuego", "AFIRMATIVO"], ["BIANCHI, María del Carmen", "Frente para la Victoria - PJ", "Cdad. Aut. Bs. As.", "AFIRMATIVO"], ["BIDEGAIN, Gloria Mercedes", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["BRAWER, Mara", "Frente para la Victoria - PJ", "Cdad. Aut. Bs. As.", "AFIRMATIVO"], ["BRILLO, José Ricardo", "Movimiento Popular Neuquino", "Neuquén", "AFIRMATIVO"], ["BROMBERG, Isaac Benjamín", "Frente para la Victoria - PJ", "Tucumán", "AFIRMATIVO"], ["BRUE, Daniel Agustín", "Frente Cívico por Santiago", "Santiago del Estero", "AFIRMATIVO"], ["CALCAGNO, Eric", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["CARLOTTO, Remo Gerardo", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["CARMONA, Guillermo Ramón", "Frente para la Victoria - PJ", "Mendoza", "AFIRMATIVO"], ["CATALAN MAGNI, Julio César", "Frente para la Victoria - PJ", "Tierra del Fuego", "AFIRMATIVO"], ["CEJAS, Jorge Alberto", "Frente para la Victoria - PJ", "Rio Negro", "AFIRMATIVO"], ["CHIENO, María Elena", "Frente para la Victoria - PJ", "Corrientes", "AFIRMATIVO"], ["CIAMPINI, José Alberto", "Frente para la Victoria - PJ", "Neuquén", "AFIRMATIVO"], ["CIGOGNA, Luis Francisco Jorge", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["CLERI, Marcos", "Frente para la Victoria - PJ", "Santa Fe", "AFIRMATIVO"], ["COMELLI, Alicia Marcela", "Movimiento Popular Neuquino", "Neuquén", "AFIRMATIVO"], ["CONTI, Diana Beatriz", "Frente para la Victoria - PJ", "Buenos Aires", "AFIRMATIVO"], ["CORDOBA, Stella Maris", "Frente para la Victoria - PJ", "Tucumán", "AFIRMATIVO"], ["CURRILEN, Oscar Rubén", "Frente para la Victoria - PJ", "Chubut", "AFIRMATIVO"]] assert_equal expected, table end - def test_forest_disclosure_report_dont_regress - # this is the current state of the expected output. Ideally the output should be like - # test_forest_disclosure_report, with spaces around the & in Regional Pulmonary & Sleep - # and a solution for half-x-height-offset lines. - pdf_file_path = File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) - - table = Tabula.extract_table(pdf_file_path, - 1, - [106.01, 48.09, 227.31, 551.89], - :detect_ruling_lines => true, - :extraction_method => "original") - - expected = Tabula::Table.new_from_array([["AANONSEN, DEBORAH, A", "", "STATEN ISLAND, NY", "MEALS", "$85.00"], ["TOTAL", "", "", "", "$85.00"], ["AARON, CAREN, T", "", "RICHMOND, VA", "EDUCATIONAL ITEMS", "$78.80"], ["AARON, CAREN, T", "", "RICHMOND, VA", "MEALS", "$392.45"], ["TOTAL", "", "", "", "$471.25"], ["AARON, JOHN", "", "CLARKSVILLE, TN", "MEALS", "$20.39"], ["TOTAL", "", "", "", "$20.39"], ["AARON, JOSHUA, N", "", "WEST GROVE, PA", "MEALS", "$310.33"], ["", "REGIONAL PULMONARY & SLEEP"], ["AARON, JOSHUA, N", "", "WEST GROVE, PA", "SPEAKING FEES", "$4,700.00"], ["", "MEDICINE"], ["TOTAL", "", "", "", "$5,010.33"], ["AARON, MAUREEN, M", "", "MARTINSVILLE, VA", "MEALS", "$193.67"], ["TOTAL", "", "", "", "$193.67"], ["AARON, MICHAEL, L", "", "WEST ISLIP, NY", "MEALS", "$19.50"], ["TOTAL", "", "", "", "$19.50"], ["AARON, MICHAEL, R", "", "BROOKLYN, NY", "MEALS", "$65.92"]]) - - assert_equal expected, table - end - def test_missing_spaces_around_an_ampersand pdf_file_path = File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) character_extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path) - page_obj = character_extractor.extract.next + page_obj = character_extractor.extract.first lines = page_obj.ruling_lines vertical_rulings = lines.select(&:vertical?) - area = [170, 28, 185, 833] #top left bottom right + top, left, bottom, right = [170, 28, 185, 833] #top left bottom right expected = Tabula::Table.new_from_array([ ["", "REGIONAL PULMONARY & SLEEP",], @@ -229,36 +219,8 @@ def test_missing_spaces_around_an_ampersand ["", "MEDICINE", ], ]) - assert_equal expected, lines_to_table(page_obj.get_area(area).make_table(:vertical_rulings => vertical_rulings)) - end - - def test_forest_disclosure_report - skip "Skipping until we support multiline cells" - pdf_file_path = File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) - character_extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path) - lines = Tabula::TableGuesser.find_lines_on_page(pdf_file_path, 0) - vertical_rulings = lines.select(&:vertical?) #.uniq{|line| (line.left / 10).round } - - page_obj = character_extractor.extract.next - characters = page_obj.get_text([110, 28, 218, 833]) - #top left bottom right - expected = Tabula::Table.new_from_array([ - ['AANONSEN, DEBORAH, A', '', 'STATEN ISLAND, NY', 'MEALS', '', '$85.00'], - ['TOTAL', '', '', '','$85.00'], - ['AARON, CAREN, T', '', 'RICHMOND, VA', 'EDUCATIONAL ITEMS', '', '$78.80'], - ['AARON, CAREN, T', '', 'RICHMOND, VA', 'MEALS', '', '$392.45'], - ['TOTAL', '', '', '', '$471.25'], - ['AARON, JOHN', '', 'CLARKSVILLE, TN', 'MEALS', '', '$20.39'], - ['TOTAL', '', '', '','$20.39'], - ['AARON, JOSHUA, N', '', 'WEST GROVE, PA', 'MEALS', '', '$310.33'], - ['AARON, JOSHUA, N', 'REGIONAL PULMONARY & SLEEP MEDICINE', 'WEST GROVE, PA', 'SPEAKING FEES', '', '$4,700.00'], - ['TOTAL', '', '', '', '$5,010.33'], - ['AARON, MAUREEN, M', '', 'MARTINSVILLE, VA', 'MEALS', '', '$193.67'], - ['TOTAL', '', '', '', '$193.67'], - ['AARON, MICHAEL, L', '', 'WEST ISLIP, NY', 'MEALS', '', '$19.50'] - ]) - - assert_equal expected, lines_to_table(Tabula.make_table(characters, :vertical_rulings => vertical_rulings)) + assert_equal expected, lines_to_table(page_obj.get_area(top,left,bottom,right).make_table(:vertical_rulings => vertical_rulings)) + character_extractor.close! end # TODO Spaces inserted in words - fails @@ -302,19 +264,20 @@ def test_vertical_rulings_splitting_words scale_factor = pdf_page.width / 1700 - vertical_rulings = [0, 360, 506, 617, 906, 1034, 1160, 1290, 1418, 1548].map{|n| Tabula::Ruling.new(0, n * scale_factor, 0, 1000)} + vertical_rulings = [0, 360, 506, 617, 906, 1034, 1160, 1290, 1418, 1548].map{ |n| Tabula::Ruling.new(0, n * scale_factor, 0, 1000)} tables = page_areas.map do |page_area| - pdf_page.get_area(page_area).make_table(:vertical_rulings => vertical_rulings) + pdf_page.get_area(*page_area).make_table(:vertical_rulings => vertical_rulings) end assert_equal expected, lines_to_table(tables.first) end + extractor.close! end def test_vertical_rulings_prevent_merging_of_columns expected = [["SZARANGOWICZ", "GUSTAVO ALEJANDRO", "25.096.244", "20-25096244-5", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["TAILHADE", "LUIS RODOLFO", "21.386.299", "20-21386299-6", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["TEDESCHI", "ADRIÁN ALBERTO", "24.171.507", "20-24171507-9", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["URRIZA", "MARÍA TERESA", "18.135.604", "27-18135604-4", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["USTARROZ", "GERÓNIMO JAVIER", "24.912.947", "20-24912947-0", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["VALSANGIACOMO BLANC", "OFERNANDO JORGE", "26.800.203", "20-26800203-1", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["VICENTE", "PABLO ARIEL", "21.897.586", "20-21897586-1", "09/10/2013", "EFECTIVO", "$ 10.000,00"], ["AMBURI", "HUGO ALBERTO", "14.096.560", "20-14096560-0", "09/10/2013", "EFECTIVO", "$ 20.000,00"], ["BERRA", "CLAUDIA SUSANA", "14.433.112", "27-14433112-0", "09/10/2013", "EFECTIVO", "$ 10.000,00"]] - vertical_rulings = [47,147,256,310,375,431,504].map{|n| Tabula::Ruling.new(0, n, 0, 1000)} + vertical_rulings = [ 147, 256, 310, 375, 431, 504].map{ |n| Tabula::Ruling.new(0, n, 0, 1000) } table = table_to_array Tabula.extract_table(File.expand_path('data/campaign_donors.pdf', File.dirname(__FILE__)), 1, @@ -345,91 +308,6 @@ def initialize(cells) end end - #just tests the algorithm - def test_cells_to_spreadsheets - - cells = [Tabula::Cell.new(40.0, 18.0, 208.0, 4.0), Tabula::Cell.new(44.0, 18.0, 52.0, 6.0), - Tabula::Cell.new(50.0, 18.0, 52.0, 4.0), Tabula::Cell.new(54.0, 18.0, 52.0, 6.0), - Tabula::Cell.new(60.0, 18.0, 52.0, 4.0), Tabula::Cell.new(64.0, 18.0, 52.0, 6.0), - Tabula::Cell.new(70.0, 18.0, 52.0, 4.0), Tabula::Cell.new(74.0, 18.0, 52.0, 6.0), - Tabula::Cell.new(90.0, 18.0, 52.0, 4.0), Tabula::Cell.new(94.0, 18.0, 52.0, 6.0), - Tabula::Cell.new(100.0, 18.0, 52.0, 28.0), Tabula::Cell.new(128.0, 18.0, 52.0, 4.0), - Tabula::Cell.new(132.0, 18.0, 52.0, 64.0), Tabula::Cell.new(196.0, 18.0, 52.0, 66.0), - Tabula::Cell.new(262.0, 18.0, 52.0, 4.0), Tabula::Cell.new(266.0, 18.0, 52.0, 84.0), - Tabula::Cell.new(350.0, 18.0, 52.0, 4.0), Tabula::Cell.new(354.0, 18.0, 52.0, 32.0), - Tabula::Cell.new(386.0, 18.0, 52.0, 38.0), Tabula::Cell.new(424.0, 18.0, 52.0, 18.0), - Tabula::Cell.new(442.0, 18.0, 52.0, 74.0), Tabula::Cell.new(516.0, 18.0, 52.0, 28.0), - Tabula::Cell.new(544.0, 18.0, 52.0, 4.0), Tabula::Cell.new(44.0, 70.0, 156.0, 6.0), - Tabula::Cell.new(50.0, 70.0, 156.0, 4.0), Tabula::Cell.new(54.0, 70.0, 156.0, 6.0), - Tabula::Cell.new(60.0, 70.0, 156.0, 4.0), Tabula::Cell.new(64.0, 70.0, 156.0, 6.0), - Tabula::Cell.new(70.0, 70.0, 156.0, 4.0), Tabula::Cell.new(74.0, 70.0, 156.0, 6.0), - Tabula::Cell.new(84.0, 70.0, 2.0, 6.0), Tabula::Cell.new(90.0, 70.0, 156.0, 4.0), - Tabula::Cell.new(94.0, 70.0, 156.0, 6.0), Tabula::Cell.new(100.0, 70.0, 156.0, 28.0), - Tabula::Cell.new(128.0, 70.0, 156.0, 4.0), Tabula::Cell.new(132.0, 70.0, 156.0, 64.0), - Tabula::Cell.new(196.0, 70.0, 156.0, 66.0), Tabula::Cell.new(262.0, 70.0, 156.0, 4.0), - Tabula::Cell.new(266.0, 70.0, 156.0, 84.0), Tabula::Cell.new(350.0, 70.0, 156.0, 4.0), - Tabula::Cell.new(354.0, 70.0, 156.0, 32.0), Tabula::Cell.new(386.0, 70.0, 156.0, 38.0), - Tabula::Cell.new(424.0, 70.0, 156.0, 18.0), Tabula::Cell.new(442.0, 70.0, 156.0, 74.0), - Tabula::Cell.new(516.0, 70.0, 156.0, 28.0), Tabula::Cell.new(544.0, 70.0, 156.0, 4.0), - Tabula::Cell.new(84.0, 72.0, 446.0, 6.0), Tabula::Cell.new(90.0, 226.0, 176.0, 4.0), - Tabula::Cell.new(94.0, 226.0, 176.0, 6.0), Tabula::Cell.new(100.0, 226.0, 176.0, 28.0), - Tabula::Cell.new(128.0, 226.0, 176.0, 4.0), Tabula::Cell.new(132.0, 226.0, 176.0, 64.0), - Tabula::Cell.new(196.0, 226.0, 176.0, 66.0), Tabula::Cell.new(262.0, 226.0, 176.0, 4.0), - Tabula::Cell.new(266.0, 226.0, 176.0, 84.0), Tabula::Cell.new(350.0, 226.0, 176.0, 4.0), - Tabula::Cell.new(354.0, 226.0, 176.0, 32.0), Tabula::Cell.new(386.0, 226.0, 176.0, 38.0), - Tabula::Cell.new(424.0, 226.0, 176.0, 18.0), Tabula::Cell.new(442.0, 226.0, 176.0, 74.0), - Tabula::Cell.new(516.0, 226.0, 176.0, 28.0), Tabula::Cell.new(544.0, 226.0, 176.0, 4.0), - Tabula::Cell.new(90.0, 402.0, 116.0, 4.0), Tabula::Cell.new(94.0, 402.0, 116.0, 6.0), - Tabula::Cell.new(100.0, 402.0, 116.0, 28.0), Tabula::Cell.new(128.0, 402.0, 116.0, 4.0), - Tabula::Cell.new(132.0, 402.0, 116.0, 64.0), Tabula::Cell.new(196.0, 402.0, 116.0, 66.0), - Tabula::Cell.new(262.0, 402.0, 116.0, 4.0), Tabula::Cell.new(266.0, 402.0, 116.0, 84.0), - Tabula::Cell.new(350.0, 402.0, 116.0, 4.0), Tabula::Cell.new(354.0, 402.0, 116.0, 32.0), - Tabula::Cell.new(386.0, 402.0, 116.0, 38.0), Tabula::Cell.new(424.0, 402.0, 116.0, 18.0), - Tabula::Cell.new(442.0, 402.0, 116.0, 74.0), Tabula::Cell.new(516.0, 402.0, 116.0, 28.0), - Tabula::Cell.new(544.0, 402.0, 116.0, 4.0), Tabula::Cell.new(84.0, 518.0, 246.0, 6.0), - Tabula::Cell.new(90.0, 518.0, 186.0, 4.0), Tabula::Cell.new(94.0, 518.0, 186.0, 6.0), - Tabula::Cell.new(100.0, 518.0, 186.0, 28.0), Tabula::Cell.new(128.0, 518.0, 186.0, 4.0), - Tabula::Cell.new(132.0, 518.0, 186.0, 64.0), Tabula::Cell.new(196.0, 518.0, 186.0, 66.0), - Tabula::Cell.new(262.0, 518.0, 186.0, 4.0), Tabula::Cell.new(266.0, 518.0, 186.0, 84.0), - Tabula::Cell.new(350.0, 518.0, 186.0, 4.0), Tabula::Cell.new(354.0, 518.0, 186.0, 32.0), - Tabula::Cell.new(386.0, 518.0, 186.0, 38.0), Tabula::Cell.new(424.0, 518.0, 186.0, 18.0), - Tabula::Cell.new(442.0, 518.0, 186.0, 74.0), Tabula::Cell.new(516.0, 518.0, 186.0, 28.0), - Tabula::Cell.new(544.0, 518.0, 186.0, 4.0), Tabula::Cell.new(90.0, 704.0, 60.0, 4.0), - Tabula::Cell.new(94.0, 704.0, 60.0, 6.0), Tabula::Cell.new(100.0, 704.0, 60.0, 28.0), - Tabula::Cell.new(128.0, 704.0, 60.0, 4.0), Tabula::Cell.new(132.0, 704.0, 60.0, 64.0), - Tabula::Cell.new(196.0, 704.0, 60.0, 66.0), Tabula::Cell.new(262.0, 704.0, 60.0, 4.0), - Tabula::Cell.new(266.0, 704.0, 60.0, 84.0), Tabula::Cell.new(350.0, 704.0, 60.0, 4.0), - Tabula::Cell.new(354.0, 704.0, 60.0, 32.0), Tabula::Cell.new(386.0, 704.0, 60.0, 38.0), - Tabula::Cell.new(424.0, 704.0, 60.0, 18.0), Tabula::Cell.new(442.0, 704.0, 60.0, 74.0), - Tabula::Cell.new(516.0, 704.0, 60.0, 28.0), Tabula::Cell.new(544.0, 704.0, 60.0, 4.0), - Tabula::Cell.new(84.0, 764.0, 216.0, 6.0), Tabula::Cell.new(90.0, 764.0, 216.0, 4.0), - Tabula::Cell.new(94.0, 764.0, 216.0, 6.0), Tabula::Cell.new(100.0, 764.0, 216.0, 28.0), - Tabula::Cell.new(128.0, 764.0, 216.0, 4.0), Tabula::Cell.new(132.0, 764.0, 216.0, 64.0), - Tabula::Cell.new(196.0, 764.0, 216.0, 66.0), Tabula::Cell.new(262.0, 764.0, 216.0, 4.0), - Tabula::Cell.new(266.0, 764.0, 216.0, 84.0), Tabula::Cell.new(350.0, 764.0, 216.0, 4.0), - Tabula::Cell.new(354.0, 764.0, 216.0, 32.0), Tabula::Cell.new(386.0, 764.0, 216.0, 38.0), - Tabula::Cell.new(424.0, 764.0, 216.0, 18.0), Tabula::Cell.new(442.0, 764.0, 216.0, 74.0), - Tabula::Cell.new(516.0, 764.0, 216.0, 28.0), Tabula::Cell.new(544.0, 764.0, 216.0, 4.0)] - - - expected_spreadsheets = [Tabula::Spreadsheet.new(40.0, 18.0, 208.0, 40.0, nil, nil, nil, nil), - Tabula::Spreadsheet.new(84.0, 18.0, 962.0, 464.0,nil, nil, nil, nil)] - - #compares spreadsheets on area only. - assert_equal expected_spreadsheets.map{|s| [s.x, s.y, s.width, s.height] }, - SpreadsheetsHasCellsTester.new(cells).find_spreadsheets_from_cells.map{|a| s = a.getBounds; [s.x, s.y, s.width, s.height] } - - - end - - def test_add_spanning_cells - skip "until I write it" - end - - def test_add_placeholder_cells_to_funny_shaped_tables - skip "until I write it, cf 01005787B_Pakistan.pdf" - end - class CellsHasCellsTester include Tabula::HasCells attr_accessor :vertical_ruling_lines, :horizontal_ruling_lines, :cells @@ -437,7 +315,7 @@ def initialize(vertical_ruling_lines, horizontal_ruling_lines) @cells = [] @vertical_ruling_lines = vertical_ruling_lines @horizontal_ruling_lines = horizontal_ruling_lines - find_cells! + find_cells!(horizontal_ruling_lines, vertical_ruling_lines) end end @@ -472,65 +350,76 @@ def test_lines_to_cells #this is the real deal!! def test_extract_tabular_data_using_lines_and_spreadsheets - pdf_file_path = "./test/data/frx_2012_disclosure.pdf" - expected_data_path = "./test/data/frx_2012_disclosure.tsv" - expected = open(expected_data_path, 'r').read #.split("\n").map{|line| line.split("\t")} + pdf_file_path = File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) + expected_data_path = File.expand_path('data/frx_2012_disclosure.tsv', File.dirname(__FILE__)) + expected = open(expected_data_path, 'r').read - Tabula::Extraction::ObjectExtractor.new(pdf_file_path, :all).extract.each do |pdf_page| + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, :all) + extractor.extract.each do |pdf_page| spreadsheet = pdf_page.spreadsheets.first assert_equal expected, spreadsheet.to_tsv end + extractor.close! end def test_cope_with_a_tableless_page - pdf_file_path = "./test/data/no_tables.pdf" - - spreadsheets = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, :all, '', - :line_color_filter => lambda{|components| components.all?{|c| c < 0.1}} - ).extract.to_a.first.spreadsheets - + skip("line_color_filter unimplemented in tabula-java for now, see https://github.com/tabulapdf/tabula-java/issues/21") + pdf_file_path = File.expand_path('data/no_tables.pdf', File.dirname(__FILE__)) + + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, :all, '', + :line_color_filter => lambda{|components| components.all?{|c| c < 0.1}} + ) + spreadsheets = extractor.extract.to_a.first.spreadsheets + extractor.close! assert_equal 0, spreadsheets.size end def test_spanning_cells - pdf_file_path = "./test/data/spanning_cells.pdf" - expected_data_path = "./test/data/spanning_cells.csv" + pdf_file_path = File.expand_path('data/spanning_cells.pdf', File.dirname(__FILE__)) + expected_data_path = File.expand_path('data/spanning_cells.csv', File.dirname(__FILE__)) expected = open(expected_data_path, 'r').read - - Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]).extract.each do |pdf_page| + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + extractor.extract.each do |pdf_page| spreadsheet = pdf_page.spreadsheets.first assert_equal expected, spreadsheet.to_csv end + extractor.close! end def test_almost_vertical_lines - pdf_file_path = "./test/data/puertos1.pdf" + pdf_file_path = File.expand_path('data/puertos1.pdf', File.dirname(__FILE__)) top, left, bottom, right = 273.9035714285714, 30.32142857142857, 554.8821428571429, 546.7964285714286 - area = Tabula::ZoneEntity.new(top, left, - right - left, bottom - top) + area = Rectangle.new(top, left, + right - left, bottom - top) - Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]).extract.each do |pdf_page| + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + extractor.extract.each do |pdf_page| rulings = Tabula::Ruling.crop_rulings_to_area(pdf_page.ruling_lines, area) # TODO assertion not entirely correct, should do the trick for now assert_equal 15, rulings.select(&:vertical?).count end + extractor.close! end def test_extract_spreadsheet_within_an_area - pdf_file_path = "./test/data/puertos1.pdf" - top, left, bottom, right = 273.9035714285714, 30.32142857142857, 554.8821428571429, 546.7964285714286 + pdf_file_path = File.expand_path('data/puertos1.pdf', File.dirname(__FILE__)) + top, left, bottom, right = 273.9035714285714,30.32142857142857,554.8821428571429,546.7964285714286 - Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]).extract.each do |pdf_page| - area = pdf_page.get_area([top, left, bottom, right]) - table = area.spreadsheets.first.to_a - assert_equal 15, table.length - assert_equal ["", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM"], table.first - assert_equal ["TOTAL", "453,515", "895,111", "456,431", "718,382", "487,183", "886,211", "494,220", "816,623", "495,580", "810,565", "627,469", "1,248,804", "540,367"], table.last - end + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + pdf_page = extractor.extract.first + + area = pdf_page.get_area(top, left, bottom, right) + table = area.spreadsheets.first.getRows.map { |r| r.map(&:getText) } + + assert_equal 15, table.length + assert_equal ["", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM", "M.U$S", "TM"], table.first + assert_equal ["TOTAL", "453,515", "895,111", "456,431", "718,382", "487,183", "886,211", "494,220", "816,623", "495,580", "810,565", "627,469", "1,248,804", "540,367"], table.last + + extractor.close! end def test_remove_repeated_text - top, left, bottom, right = 106.07142857142858, 50.91428571428572, 141.42857142857144, 755.2285714285715 + top, left, bottom, right = 101.82857142857144,48.08571428571429,497.8285714285715,765.1285714285715 table = Tabula.extract_table(File.expand_path('data/nyc_2013fiscalreporttables.pdf', File.dirname(__FILE__)), 1, @@ -539,13 +428,14 @@ def test_remove_repeated_text :extraction_method => 'original') ary = table_to_array(table) - assert_equal ary[1][1], "$ 18,969,610" - assert_equal ary[1][2], "$ 18,157,722" + + assert_equal "$ 18,969,610", ary[1][1] + assert_equal "$ 18,157,722", ary[1][2] end def test_remove_overlapping_text # one of those PDFs that put characters on top of another to make text "bold" - top,left,bottom,right = 399.98571428571427, 36.06428571428571, 425.1214285714285, 544.2428571428571 + top,left,bottom,right = 399.98571428571427,36.06428571428571,425.1214285714285,544.2428571428571 table = Tabula.extract_table(File.expand_path('data/wc2012.pdf', File.dirname(__FILE__)), 1, [top,left,bottom,right], @@ -558,21 +448,26 @@ def test_remove_overlapping_text def test_cells_including_line_returns data = [] - pdf_file_path = "./test/data/sydney_disclosure_contract.pdf" - Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]).extract.each do |pdf_page| + pdf_file_path = File.expand_path('data/sydney_disclosure_contract.pdf', File.dirname(__FILE__)) + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + extractor.extract([1.to_java(:int)]).each do |pdf_page| pdf_page.spreadsheets.each do |spreadsheet| spreadsheet.cells.each do |cell| - cell.text_elements = pdf_page.get_cell_text(cell) + + # this pattern is deprecated and maintained only for backwards-compatibility. please don't copy it. + # data << cell.text(true) would be sufficient in modern Tabula for all three following lines. cell.options = ({:use_line_returns => true, :cell_debug => 0}) data << cell.text end end end - assert_equal ["1295", "Name: Reino International Pty Ltd trading as Duncan Solutions \rAddress: 15/39 Herbet Street, St Leonards NSW 2065", "N/A", "Effective Date: 13 May 2013 \rDuration: 15 Weeks", "Supply, Installation and Maintenance of Parking Ticket Machines", "$3,148,800.00exgst", "N/A", "N/A", "Open Tender \rTender evaluation criteria included: \r- The schedule of prices \r- Compliance with technical specifications/Technical assessment \r- Operational Plan including maintenance procedures"], data + extractor.close! + expected = ["1295", "Name: Reino International Pty Ltd trading as Duncan Solutions \rAddress: 15/39 Herbet Street, St Leonards NSW 2065", "N/A", "Effective Date: 13 May 2013 \rDuration: 15 Weeks", "Supply, Installation and Maintenance of Parking Ticket Machines", "$3,148,800.00exgst", "N/A", "N/A", "Open Tender \rTender evaluation criteria included: \r- The schedule of prices \r- Compliance with technical specifications/Technical assessment \r- Operational Plan including maintenance procedures"] + assert_equal expected, data end def test_remove_repeated_spaces - top,left,bottom,right = 304.9375, 78.625, 334.6875, 501.5 + top,left,bottom,right = 304.9375,78.625,334.6875,501.5 table = Tabula.extract_table(File.expand_path('data/repeated_spaces.pdf', File.dirname(__FILE__)), 1, [top,left,bottom,right], @@ -593,28 +488,98 @@ def test_monospaced_table :extraction_method => 'original') expected = [["ALBERT LEA, MAYO CLINIC HEALTH SYS- ALBE", "0", "0", "0", "7", "7", ".0", ".0", ".0", "23.3", "10.4"], ["ROCHESTER, MAYO CLINIC METHODIST HOSPITA", "6", "7", "14", "11", "25", "27.3", "100.0", "37.8", "36.7", "37.3"], ["ROCHESTER, MAYO CLINIC ST. MARYS", "9", "0", "11", "7", "18", "40.9", ".0", "29.7", "23.3", "26.9"], ["BLUE EARTH, UNITED HOSPITAL DISTRICT", "3", "0", "4", "0", "4", "13.6", ".0", "10.8", ".0", "6.0"], ["FAIRMONT, MAYO CLINIC HEALTH SYSTEM -FAI", "1", "0", "2", "1", "3", "4.5", ".0", "5.4", "3.3", "4.5"], ["MANKATO, MAYO CLINIC HEALTH SYSTEM- MANK", "3", "0", "5", "3", "8", "13.6", ".0", "13.5", "10.0", "11.9"], ["ALL REGION 4 (TC) HOSPITALS", "0", "0", "1", "1", "2", ".0", ".0", "2.7", "3.3", "3.0"], ["", "22", "7", "37", "30", "67", "100.0", "100.0", "100.0", "100.0", "100.0"]] - assert_equal table_to_array(table), expected + assert_equal expected, table_to_array(table) + end + + def test_monospaced_table_ascii_line_separator + extractor = Tabula::Extraction::ObjectExtractor.new(File.expand_path('data/monospaced_ascii_sep.pdf', File.dirname(__FILE__)), + [1]) + expected = [["Column A", "* ColB1", "ColB2 ColB3", "* Column C"], ["Value 1", "* 23.5", "66.811.0", "* Name 1"], ["Value 2", "* 33.2", "56.312.0", "* Name 2"], ["Value 3", "* 123.3", "200.4 123.9", "* Name 3"], ["Value 4", "* 24.5", "66.811.0", "* Name 1"], ["Value 5", "* 43.2", "80.114.5", "* Name 2"], ["Value 6", "* 100.6", "190.4 120.3", "* Name 3"], ["Value 7", "* 11.5", "66.811.0", "* Name 1"], ["Value 8", "* 37.4", "77.420.1", "* Name 2"], ["Value 9", "* 883.3", "110.4 111.2", "* Name 3"]] + + table = extractor.extract_page(1).get_table + assert_equal expected, table_to_array(table) end + def test_bad_column_detection - top,left,bottom,right = 535.5, 70.125, 549.3125, 532.3125 + top,left,bottom,right = 535.5,70.125,549.3125,532.3125 table = Tabula.extract_table(File.expand_path('data/indecago10.pdf', File.dirname(__FILE__)), 1, [top,left,bottom,right], :detect_ruling_lines => false, :extraction_method => 'original') - assert_equal table_to_array(table).first, ["Comunicaciones", "104,29", "– –", "0,1", "0,6", "1,1", "0,3"] + assert_equal ["Comunicaciones", "104,29", "– –", "0,1", "0,6", "1,1", "0,3"], + table_to_array(table).first end + def test_character_merging_that_wasnt_working_previously + expected_data_path = File.expand_path('data/french1.tsv', File.dirname(__FILE__)) + expected = CSV.read(expected_data_path, { :col_sep => "\t" }) + expected.map! { |r| r.map(&:strip) } + top,left,bottom,right = 32.87142857142857,41.72142857142857,486.75,694.0928571428572 + table = Tabula.extract_table(File.expand_path('data/french1.pdf', File.dirname(__FILE__)), + 1, + [top,left,bottom,right], + :detect_ruling_lines => false, + :extraction_method => 'original') + + assert_equal expected, table_to_array(table) + end + + def test_issue78_some_ruling_lines_not_detected + pdf_file_path = File.expand_path('data/mineria.pdf', File.dirname(__FILE__)) + area = [104.46890818740722, 13, 580.548646927163, 820.8271357581996] + table = Tabula.extract_table(File.expand_path('data/mineria.pdf', + File.dirname(__FILE__)), + 1, + area, + :extraction_method => 'spreadsheet') + expected = [["1", "010000091", "086", "03/12/2012", "ACHAYAP MANTU ALDO", "ACHAYAP MANTU ALDO", "1", "OTROS", ".", ".", "AMAZONAS", "CONDORCANQ\rUI", "NIEVA", "", "", ""], ["2", "010000023", "022", "18/06/2012", "ACOSTA ROSALES YOSELIN BRICET", "ACOSTA ROSALES YOSELIN \rBRICET", "2", "TITULAR", "NANCY 11 ( REGISTRO \rCANCELADO )", "510001910", "AMAZONAS", "BONGARA", "FLORIDA", "18", "9,357,000", "174,000"]] + assert expected, table_to_array(table)[0..2] + end + + + def test_checks_for_text_on_page + pdf_file_path = File.expand_path('data/gretna-owh-request.pdf', File.dirname(__FILE__)) + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + extractor.extract.each do |pdf_page| + assert !pdf_page.has_text? + end + extractor.close! + + pdf_file_path = File.expand_path('data/brazil_crop_area.pdf', File.dirname(__FILE__)) + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path, [1]) + extractor.extract.each do |pdf_page| + assert pdf_page.has_text? + end + extractor.close! + end + + def test_spanning_cells + # see + # https://github.com/tabulapdf/tabula-java/issues/55 + # https://github.com/tabulapdf/tabula-extractor/commit/735b82450a40b3743333816b50cad470b1ca7b43#commitcomment-15087511 + # this PDF is a printout of an Excel spreadsheet that has "spanning cells", that is, 10 or so data columns + # with six header cells -- some of the header cells are "merged" to span over several data columns. + # we cope with this internally by creating zero-width (or zero-height for cells that span rows) cells + # and in array/CSV output, these blank cells are inserted /after/ the text of the real cell. + pdf_file_path = File.expand_path("data/47008204D_USA.page4.pdf", File.dirname(__FILE__)) + extractor = Tabula::Extraction::ObjectExtractor.new(pdf_file_path) + pdf_page = extractor.extract.first + actual_header_row = table_to_array(pdf_page.spreadsheets.first).first + expected_header_row = ["", "", "", "IEM Findings","","","","","", "Remediation","","","", "[Status]"] + assert_equal expected_header_row, actual_header_row + end end class TestIsTabularHeuristic < Minitest::Test - EXPECTED_TO_BE_SPREADSHEET = ['47008204D_USA.page4.pdf', 'GSK_2012_Q4.page437.pdf', 'strongschools.pdf', 'tabla_subsidios.pdf'] - NOT_EXPECTED_TO_BE_SPREADSHEET = ['560015757GV_China.page1.pdf', 'S2MNCEbirdisland.pdf', 'bo_page24.pdf', 'campaign_donors.pdf'] + EXPECTED_TO_BE_SPREADSHEET = ['GSK_2012_Q4.page437.pdf', 'strongschools.pdf', 'tabla_subsidios.pdf'] + #NOT_EXPECTED_TO_BE_SPREADSHEET = ['560015757GV_China.page1.pdf', 'S2MNCEbirdisland.pdf', 'bo_page24.pdf', 'campaign_donors.pdf'] + NOT_EXPECTED_TO_BE_SPREADSHEET = ['47008204D_USA.page4.pdf', '560015757GV_China.page1.pdf', 'bo_page24.pdf', 'campaign_donors.pdf'] File.expand_path('data/frx_2012_disclosure.pdf', File.dirname(__FILE__)) @@ -624,6 +589,7 @@ def test_heuristic_detects_spreadsheets extractor = Tabula::Extraction::ObjectExtractor.new(path, [1]) page = extractor.extract.first page.get_ruling_lines! + extractor.close! assert page.is_tabular?, "failed on file #{f}" end end @@ -634,6 +600,7 @@ def test_heuristic_detects_non_spreadsheets extractor = Tabula::Extraction::ObjectExtractor.new(path, [1]) page = extractor.extract.first page.get_ruling_lines! + extractor.close! assert !page.is_tabular?, "failed on file #{f}" end end